Somasundaram Ayyappan Claude Opus 4.6 (1M context) commited on
Commit
1652579
·
1 Parent(s): 03116a3

Add noise augmentation for OCR robustness training

Browse files

Generates 2 noisy variants per training example by applying:
- Separator swaps (| ↔ + ↔ - ↔ ● at 20% per token)
- Character corruption on O-tagged tokens (5%) and entities (3%)
- Token merging to simulate missing spaces (3%)
- Random case changes on non-entity tokens (5%)
- Random line break insertion (2%)

Training data grows from 3647 → 10941 with augmentation.
BIO tag alignment preserved through all transforms.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

training/generate_from_structured.py CHANGED
@@ -280,6 +280,15 @@ def main():
280
 
281
  clean, duplicates_removed = dedupe_examples(clean)
282
  train, val = stable_split_examples(clean, train_ratio=0.85)
 
 
 
 
 
 
 
 
 
283
  write_dataset(
284
  train,
285
  val,
@@ -292,6 +301,7 @@ def main():
292
  "manual_templates": len(manual),
293
  "long_resumes": len(long_resumes),
294
  "resume_resource": len(resume_resource),
 
295
  "duplicates_removed": duplicates_removed,
296
  },
297
  },
 
280
 
281
  clean, duplicates_removed = dedupe_examples(clean)
282
  train, val = stable_split_examples(clean, train_ratio=0.85)
283
+
284
+ try:
285
+ from training.noise_augment import augment_examples
286
+ except ModuleNotFoundError:
287
+ from noise_augment import augment_examples
288
+
289
+ augmented = augment_examples(train, multiplier=2, seed=42)
290
+ print(f"Noise augmented: {len(augmented)} (from {len(train)} train examples)")
291
+ train = train + augmented
292
  write_dataset(
293
  train,
294
  val,
 
301
  "manual_templates": len(manual),
302
  "long_resumes": len(long_resumes),
303
  "resume_resource": len(resume_resource),
304
+ "noise_augmented": len(augmented),
305
  "duplicates_removed": duplicates_removed,
306
  },
307
  },
training/noise_augment.py ADDED
@@ -0,0 +1,153 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Noise augmentation for training data to improve OCR robustness.
2
+
3
+ Generates noisy variants of clean training examples by applying
4
+ random perturbations that mimic real-world OCR and PDF extraction
5
+ artifacts. Each augmented example preserves BIO tag alignment.
6
+
7
+ Usage:
8
+ from training.noise_augment import augment_examples
9
+ noisy = augment_examples(clean_examples, multiplier=2)
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import random
15
+ import re
16
+ import string
17
+
18
+
19
+ def _swap_separator(token: str) -> str:
20
+ """Randomly swap separator characters: | + - ●"""
21
+ seps = {"|": ["+", "-", "●", "•"], "+": ["|", "-", "•"], "-": ["|", "+"], "●": ["|", "+", "•"], "•": ["|", "+", "●"]}
22
+ if token in seps:
23
+ return random.choice(seps[token])
24
+ return token
25
+
26
+
27
+ def _corrupt_char(token: str) -> str:
28
+ """Drop, swap, or insert a random character to mimic OCR errors."""
29
+ if len(token) < 3:
30
+ return token
31
+ i = random.randint(0, len(token) - 1)
32
+ action = random.choice(["drop", "swap", "insert"])
33
+ if action == "drop":
34
+ return token[:i] + token[i + 1:]
35
+ elif action == "swap" and i < len(token) - 1:
36
+ return token[:i] + token[i + 1] + token[i] + token[i + 2:]
37
+ else:
38
+ c = random.choice(string.ascii_lowercase)
39
+ return token[:i] + c + token[i:]
40
+
41
+
42
+ def _merge_tokens(tokens: list[str], labels: list[str], idx: int) -> bool:
43
+ """Merge token at idx with next token (remove space between them)."""
44
+ if idx >= len(tokens) - 1:
45
+ return False
46
+ if labels[idx] != "O" or labels[idx + 1] != "O":
47
+ return False
48
+ tokens[idx] = tokens[idx] + tokens[idx + 1]
49
+ tokens.pop(idx + 1)
50
+ labels.pop(idx + 1)
51
+ return True
52
+
53
+
54
+ def augment_one(tokens: list[str], labels: list[str], seed: int | None = None) -> tuple[list[str], list[str]]:
55
+ """Create one noisy variant of a training example.
56
+
57
+ Applies random subset of noise operations while preserving
58
+ BIO tag alignment. Only modifies O-tagged tokens for safety,
59
+ except separator swaps which can happen anywhere.
60
+ """
61
+ if seed is not None:
62
+ random.seed(seed)
63
+
64
+ tok = list(tokens)
65
+ lab = list(labels)
66
+
67
+ # 1. Swap separators (20% per separator token)
68
+ for i in range(len(tok)):
69
+ if tok[i] in ("|", "+", "-", "●", "•") and random.random() < 0.2:
70
+ tok[i] = _swap_separator(tok[i])
71
+
72
+ # 2. Corrupt characters in O-tagged tokens (5% chance per token)
73
+ for i in range(len(tok)):
74
+ if lab[i] == "O" and len(tok[i]) >= 3 and random.random() < 0.05:
75
+ tok[i] = _corrupt_char(tok[i])
76
+
77
+ # 3. Corrupt characters in entity tokens (3% chance — lower rate to keep most entities clean)
78
+ for i in range(len(tok)):
79
+ if lab[i] != "O" and len(tok[i]) >= 4 and random.random() < 0.03:
80
+ tok[i] = _corrupt_char(tok[i])
81
+
82
+ # 4. Merge adjacent O tokens (3% chance per pair)
83
+ i = 0
84
+ while i < len(tok) - 1:
85
+ if random.random() < 0.03:
86
+ _merge_tokens(tok, lab, i)
87
+ else:
88
+ i += 1
89
+
90
+ # 5. Random case change on O tokens (5% chance)
91
+ for i in range(len(tok)):
92
+ if lab[i] == "O" and random.random() < 0.05:
93
+ if random.random() < 0.5:
94
+ tok[i] = tok[i].upper()
95
+ else:
96
+ tok[i] = tok[i].lower()
97
+
98
+ # 6. Insert random line-break token between O tokens (2% chance)
99
+ inserts = []
100
+ for i in range(len(tok) - 1):
101
+ if lab[i] == "O" and lab[i + 1] == "O" and random.random() < 0.02:
102
+ inserts.append(i + 1)
103
+ for offset, idx in enumerate(inserts):
104
+ tok.insert(idx + offset, "\n")
105
+ lab.insert(idx + offset, "O")
106
+
107
+ return tok, lab
108
+
109
+
110
+ def augment_examples(
111
+ examples: list[dict],
112
+ multiplier: int = 2,
113
+ seed: int = 42,
114
+ ) -> list[dict]:
115
+ """Generate noisy variants of training examples.
116
+
117
+ Args:
118
+ examples: list of {"tokens": [...], "ner_tags": [...], ...}
119
+ multiplier: how many noisy copies per clean example
120
+ seed: random seed for reproducibility
121
+
122
+ Returns:
123
+ List of augmented examples (does NOT include originals)
124
+ """
125
+ try:
126
+ from training.labels import ID2LABEL, LABEL2ID
127
+ except ModuleNotFoundError:
128
+ from labels import ID2LABEL, LABEL2ID
129
+
130
+ random.seed(seed)
131
+ augmented = []
132
+
133
+ for ex_idx, example in enumerate(examples):
134
+ tokens = example["tokens"]
135
+ labels = [ID2LABEL[tag] for tag in example["ner_tags"]]
136
+
137
+ for copy_idx in range(multiplier):
138
+ aug_tokens, aug_labels = augment_one(
139
+ tokens, labels,
140
+ seed=seed + ex_idx * 1000 + copy_idx,
141
+ )
142
+ aug_example = {
143
+ "tokens": aug_tokens,
144
+ "ner_tags": [LABEL2ID[l] for l in aug_labels],
145
+ }
146
+ if "metadata" in example:
147
+ meta = dict(example["metadata"])
148
+ meta["source_id"] = f"{meta.get('source_id', '')}:aug{copy_idx}"
149
+ meta["group_id"] = meta.get("group_id", "")
150
+ aug_example["metadata"] = meta
151
+ augmented.append(aug_example)
152
+
153
+ return augmented