fix(detector): label unsegmented-script documents Unsupported script (#319)

Closes #241.

CHANGELOG.md

@@ -18,6 +18,7 @@ All notable changes to this project are documented here.
 
 ### Fixed
 
+- Detect unsegmented-script documents (Chinese/Japanese: no inter-word spaces) before the word gate and label them `Unsupported script` instead of `Too short`, with the reason and CJK character count in `stats`. The check recognizes Han and kana ranges (including halfwidth katakana) and declines only when CJK characters dominate the non-whitespace text, so newline-wrapped lines cannot bypass it and short English documents with an incidental place name stay scorable. The gate CLI now exits 2 on such files — matching the documented unscannable-input exit code — instead of passing silently at every threshold, and the repository self-scan reports declined documents instead of scoring them as clean (#241).
 - Align false-positive preprocessing with CommonMark for backtick fence info strings and multiline setext headings, preserve unique normalized units as modified when only whitespace boundaries move their source spans, reject Windows OpenCode command shims with an actionable native-binary error, and recognize first-person `I` inside otherwise targeted Title Case headings (#314).
 - Restrict Title Case header word separators and trailing whitespace to horizontal whitespace, so a match can never run past one physical line. `\s` also ate newlines, which let two unrelated lines or a blank-line-separated fragment combine into a single heading match that neither line independently satisfied (#291).
 - Report the underlying OpenCode export launch error instead of a secondary `stderr.trim()` exception during rewrite evaluation.

bin/avoid-ai-writing-gate.js

@@ -129,6 +129,15 @@ function main(argv) {
       );
       return 2;
     }
+    if (result.unsupportedScript) {
+      // An unsegmented-script document (Chinese/Japanese: no inter-word
+      // spaces) was declined, not scored. README classifies unscannable
+      // input as exit 2, so the gate must not report green on it. (GH-241)
+      process.stderr.write(
+        `avoid-ai-writing-gate: cannot scan ${file}: unsegmented-script document (no inter-word spaces to count)\n`
+      );
+      return 2;
+    }
     const count = result.issues.length;
     const types = [...new Set(result.issues.map((issue) => issue.type))].sort();
     const over = count > parsed.threshold;

bin/avoid-ai-writing-gate.test.js

@@ -59,6 +59,24 @@ assert.strictEqual(oversized.status, 2, oversized.stderr);
 assert.match(oversized.stderr, /detector limit exceeded/);
 assert.doesNotMatch(oversized.stdout, /^PASS /m);
 
+// An unsegmented-script document (Chinese/Japanese: no inter-word spaces) is
+// declined, not scored; the gate must exit 2 rather than pass silently (#241).
+const cjk = path.join(tmp, "cjk.md");
+fs.writeFileSync(cjk, "这个函数返回一个承诺，调用方不应假设句柄之后仍可重用。".repeat(50), "utf8");
+const cjkRun = run([cjk]);
+assert.strictEqual(cjkRun.status, 2, cjkRun.stderr);
+assert.match(cjkRun.stderr, /unsegmented-script document/);
+assert.doesNotMatch(cjkRun.stdout, /^PASS /m);
+
+// A short English document with an incidental CJK place name is not an
+// unsegmented-script document: the dominance check keeps it scorable, so
+// the gate must not exit 2 on it (#241 review follow-up).
+const mixed = path.join(tmp, "mixed.md");
+fs.writeFileSync(mixed, "The Tokyo (東京) office owns the retry limit docs.", "utf8");
+const mixedRun = run([mixed]);
+assert.notStrictEqual(mixedRun.status, 2, mixedRun.stderr);
+assert.doesNotMatch(mixedRun.stderr, /cannot scan/);
+
 const gitRepo = path.join(tmp, "repo");
 fs.mkdirSync(gitRepo);
 spawnSync("git", ["init", "-q"], { cwd: gitRepo });

detector/README.md

@@ -53,7 +53,7 @@ Exit codes:
 
 - `0`: every scanned file is at or below the finding threshold;
 - `1`: at least one file exceeds the threshold;
-- `2`: usage, glob-expansion, file-read, UTF-8, or unscannable-input error (including documents above the detector's 10,000-word limit).
+- `2`: usage, glob-expansion, file-read, UTF-8, or unscannable-input error (including documents above the detector's 10,000-word limit and unsegmented-script documents the engine declined to score).
 
 The `--json` flag formats scan results as structured JSON on standard output with `schemaVersion`, per-file entries (`path`, `findings`, `pass`, `types`), and aggregates (`pass`, `totalFindings`, `failedFiles`).
 
@@ -103,7 +103,7 @@ CommonJS).
 | Field | Type | Meaning |
 |---|---|---|
 | `score` | `0–100` | 0 = clean, 100 = heavy AI |
-| `label` | string | scored: `Clean` (0) / `Minimal AI signals` (1–15) / `Some AI patterns` (16–35) / `Moderate AI signals` (36–60) / `Strong AI signals` (61–80) / `Heavy AI patterns` (81–100). Unscored: `Empty` / `Too short` / `Text too long` |
+| `label` | string | scored: `Clean` (0) / `Minimal AI signals` (1–15) / `Some AI patterns` (16–35) / `Moderate AI signals` (36–60) / `Strong AI signals` (61–80) / `Heavy AI patterns` (81–100). Unscored: `Empty` / `Too short` / `Unsupported script` / `Text too long` |
 | `issues[]` | `{type, text, severity, …}` | one entry per detected pattern; `type` keys map to [`CATEGORIES.md`](./CATEGORIES.md) |
 | `stats` | object | `wordCount`, per-tier counts, `contextMode`, `sourceMode`, masked-span counts, `denseAIVocab`, normalization flags, etc. |
 | `document_classification` | string | `HUMAN_ONLY` / `MIXED` / `AI_ONLY` (shape mirrors GPTZero for swap-in), or `UNSCORED` on the early-exit paths |
@@ -111,10 +111,15 @@ CommonJS).
 | `confidence_category` | `low` / `medium` / `high` | |
 | `highlight_sentence_for_ai` | region[] | sentence spans with source offsets + per-region score, for UI highlighting |
 
-The three unscored labels share one result shape: `score` 0,
+The four unscored labels share one result shape: `score` 0,
 `document_classification` `UNSCORED`, an even `class_probabilities` split, and
 `confidence_category` `low`. Branch on that classification rather than on the
-score, since clean text also scores 0 and is labeled `Clean`.
+score, since clean text also scores 0 and is labeled `Clean`. `Unsupported
+script` marks a document dominated by an unsegmented script (Chinese/Japanese:
+Han and kana characters, whose language has no inter-word spaces for
+`countWords` to split on) that was declined, not scored. An incidental place
+name or single Han character in otherwise English text does not qualify;
+Korean (Hangul) is space-separated and scores normally.
 
 `options.contextMode` accepts `general` (default), `technical`, `marketing`, and
 `personal`. Technical mode suppresses flags that are legitimate in code-adjacent

detector/patterns.js

@@ -1740,6 +1740,27 @@ const AIDetector = (() => {
     sourceMap = norm.sourceMap;
 
     const wordCount = countWords(text);
+    // Unsegmented-script check (GH-241): Chinese and Japanese carry no
+    // inter-word spaces, so word segmentation cannot measure them — a long
+    // document counts as one \S+ run and would misreport as "Too short",
+    // while newline-wrapped lines each count as a word and would score
+    // without segmentation. The check therefore runs before the word gate
+    // and declines only when CJK characters dominate the non-whitespace
+    // text, so short English documents with an incidental place name or
+    // single Han character stay scorable. Han, Hiragana, and Katakana
+    // ranges (including halfwidth) signal an unsegmented script; Hangul is
+    // space-separated and segments fine, so it is excluded.
+    const cjkChars = (text.match(/[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff66-\uff9d]/g) || []).length;
+    if (cjkChars > 0 && cjkChars * 2 >= (text.match(/\S/g) || []).length) {
+      return {
+        ...buildV2Defaults('UNSCORED', 'low'),
+        score: 0,
+        label: 'Unsupported script',
+        issues: [],
+        stats: { wordCount, cjkChars, reason: 'unsegmented-script document: no inter-word spaces to count', contextMode, contextModeFallback, sourceMode, sourceModeFallback, maskedFrontmatter, maskedHtmlComments },
+        unsupportedScript: true,
+      };
+    }
     if (wordCount < 10) {
       return {
         ...buildV2Defaults('UNSCORED', 'low'),

detector/patterns.test.js

@@ -2743,6 +2743,51 @@ test('reply openers and analytical framing are not reported as acknowledgment lo
   }
 });
 
+test('#241: unsegmented-script documents are declined, not scored "Too short"', () => {
+  // countWords counts \S+ runs; Chinese and Japanese carry no inter-word
+  // spaces, so segmentation cannot measure them. The script check runs
+  // before the word gate and declines only when CJK characters dominate
+  // the non-whitespace text. Han + kana ranges (including halfwidth
+  // katakana) signal an unsegmented script; Hangul is space-separated and
+  // segments fine, so it is excluded.
+  const zh = '这个函数返回一个承诺，调用方不应假设句柄之后仍可重用。'.repeat(50);
+  const rzh = AIDetector.analyzeText(zh);
+  assert.equal(rzh.label, 'Unsupported script', `expected Unsupported script, got ${rzh.label}`);
+  assert.equal(rzh.unsupportedScript, true);
+  assert.equal(rzh.document_classification, 'UNSCORED');
+  assert.ok(rzh.stats.cjkChars > 0, 'stats must carry the cjkChars count');
+  assert.match(rzh.stats.reason, /unsegmented-script/);
+
+  const ja = 'この関数はプロミスを返します。呼び出し側は、ハンドルがその後も再利用できると仮定してはいけません。'.repeat(40);
+  assert.equal(AIDetector.analyzeText(ja).label, 'Unsupported script');
+
+  // Halfwidth katakana (U+FF66–U+FF9D) is also an unsegmented script.
+  const jaHw = 'ﾃｽﾄ'.repeat(100);
+  assert.equal(AIDetector.analyzeText(jaHw).label, 'Unsupported script');
+
+  // Newline-wrapped CJK lines each count as a word, so the script check
+  // must not sit inside the minimum word-count condition.
+  const zhLines = Array(10).fill('这个函数返回一个承诺。').join('\n');
+  assert.equal(AIDetector.analyzeText(zhLines).label, 'Unsupported script');
+
+  // A genuinely short English document still reports Too short.
+  const en = AIDetector.analyzeText('Short text here.');
+  assert.equal(en.label, 'Too short');
+  assert.equal(en.unsupportedScript, undefined);
+
+  // An incidental CJK place name in a short English document is not an
+  // unsegmented-script document: the dominance check keeps it scorable.
+  const mixed = AIDetector.analyzeText('The Tokyo (東京) office owns the retry limit docs.');
+  assert.equal(mixed.label, 'Too short');
+  assert.equal(mixed.unsupportedScript, undefined);
+
+  // Korean is space-separated: it segments and scores normally.
+  const ko = '이 함수는 프라미스를 반환합니다. 호출자는 핸들이 나중에 재사용 가능하다고 가정해서는 안 됩니다. '.repeat(30);
+  const rko = AIDetector.analyzeText(ko);
+  assert.notEqual(rko.label, 'Unsupported script');
+  assert.equal(rko.unsupportedScript, undefined);
+});
+
 if (failed > 0) {
   console.error(`\n${failed} test(s) failed`);
   process.exit(1);

plugins/avoid-ai-writing/skills/avoid-ai-writing/detector/patterns.js

@@ -1740,6 +1740,27 @@ const AIDetector = (() => {
     sourceMap = norm.sourceMap;
 
     const wordCount = countWords(text);
+    // Unsegmented-script check (GH-241): Chinese and Japanese carry no
+    // inter-word spaces, so word segmentation cannot measure them — a long
+    // document counts as one \S+ run and would misreport as "Too short",
+    // while newline-wrapped lines each count as a word and would score
+    // without segmentation. The check therefore runs before the word gate
+    // and declines only when CJK characters dominate the non-whitespace
+    // text, so short English documents with an incidental place name or
+    // single Han character stay scorable. Han, Hiragana, and Katakana
+    // ranges (including halfwidth) signal an unsegmented script; Hangul is
+    // space-separated and segments fine, so it is excluded.
+    const cjkChars = (text.match(/[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff66-\uff9d]/g) || []).length;
+    if (cjkChars > 0 && cjkChars * 2 >= (text.match(/\S/g) || []).length) {
+      return {
+        ...buildV2Defaults('UNSCORED', 'low'),
+        score: 0,
+        label: 'Unsupported script',
+        issues: [],
+        stats: { wordCount, cjkChars, reason: 'unsegmented-script document: no inter-word spaces to count', contextMode, contextModeFallback, sourceMode, sourceModeFallback, maskedFrontmatter, maskedHtmlComments },
+        unsupportedScript: true,
+      };
+    }
     if (wordCount < 10) {
       return {
         ...buildV2Defaults('UNSCORED', 'low'),

scripts/self-scan-diagnostics.test.js

@@ -129,6 +129,25 @@ try {
     for (const r of rows) assert.strictEqual(r.budget, BUDGETS[r.file], `${r.file} scanned against budget ${r.budget}`);
   });
 
+  t('an unsupported-script document is declined, not scored as clean', () => {
+    const row = scanFile(fixture('cjk.md', '这个函数返回一个承诺，调用方不应假设句柄之后仍可重用。'.repeat(50)));
+    assert.strictEqual(row.declined, true);
+    assert.strictEqual(row.rawScore, 0);
+    assert.strictEqual(row.exemptIssues, 0);
+    assert.strictEqual(row.overBudget, false);
+  });
+
+  t('a chunked unsegmented-script document is declined, not scored as clean', () => {
+    // One-word-per-paragraph Chinese paragraphs take the chunked path, and
+    // every chunk is CJK-dominated: the scan must be marked declined rather
+    // than aggregated as a clean zero score.
+    const paragraphs = new Array(LONG_DOCUMENT_WORDS + 500).fill('这个函数返回一个承诺。');
+    const row = scanFile(fixture('chunked-cjk.md', paragraphs.join('\n\n')));
+    assert.ok(row.chunked >= 2, 'fixture must take the chunked path');
+    assert.strictEqual(row.declined, true);
+    assert.deepStrictEqual(row.topTypes, []);
+  });
+
   t('the over-budget diagnostic still prints none when nothing was detected', () => {
     const line = overBudgetDiagnostic({ file: 'x.md', exemptScore: 1, budget: 0, topTypes: [] });
     assert.strictEqual(line, 'x.md is over budget (1 > 0). Top categories: none');

scripts/self-scan.js

@@ -152,16 +152,30 @@ function scoreLongText(text) {
   if (current.length) chunks.push(current.join('\n\n'));
 
   const results = chunks
-    .map((chunk) => AIDetector.analyzeText(chunk))
-    .filter((r) => !r.tooShort && r.label !== 'Text too long');
+    .map((chunk) => AIDetector.analyzeText(chunk));
 
-  if (!results.length) return { score: 0, issues: 0, wordCount: 0, chunks: chunks.length, topTypes: [] };
+  // A declined (unsupported-script) chunk is not a completed scan: report
+  // the document as unscannable instead of scoring it as a clean zero (#241).
+  if (results.some((r) => r.unsupportedScript)) {
+    return {
+      declined: true,
+      score: 0,
+      issues: 0,
+      wordCount: results.reduce((sum, r) => sum + (r.stats.wordCount || 0), 0),
+      chunks: chunks.length,
+      topTypes: [],
+    };
+  }
+
+  const scored = results.filter((r) => !r.tooShort && r.label !== 'Text too long');
+
+  if (!scored.length) return { score: 0, issues: 0, wordCount: 0, chunks: chunks.length, topTypes: [] };
   return {
-    score: Math.max(...results.map((r) => r.score)),
-    issues: results.reduce((sum, r) => sum + r.issues.length, 0),
-    wordCount: results.reduce((sum, r) => sum + (r.stats.wordCount || 0), 0),
-    chunks: results.length,
-    topTypes: topTypes(results.flatMap((r) => r.issues)),
+    score: Math.max(...scored.map((r) => r.score)),
+    issues: scored.reduce((sum, r) => sum + r.issues.length, 0),
+    wordCount: scored.reduce((sum, r) => sum + (r.stats.wordCount || 0), 0),
+    chunks: scored.length,
+    topTypes: topTypes(scored.flatMap((r) => r.issues)),
   };
 }
 
@@ -169,6 +183,9 @@ function score(text) {
   const wordCount = (text.match(/\S+/g) || []).length;
   if (wordCount > 9500) return scoreLongText(text);
   const r = AIDetector.analyzeText(text);
+  if (r.unsupportedScript) {
+    return { declined: true, score: 0, issues: 0, wordCount: r.stats.wordCount || wordCount, chunks: 1, topTypes: [] };
+  }
   return {
     score: r.score,
     issues: r.issues.length,
@@ -200,6 +217,7 @@ function scanFile(rel, budget = BUDGETS[rel]) {
     exemptScore: exempt.score,
     exemptIssues: exempt.issues,
     budget,
+    declined: raw.declined || exempt.declined || null,
     overBudget: exempt.score > budget,
     chunked: raw.chunks > 1 ? raw.chunks : null,
     topTypes: exempt.topTypes || [],
@@ -222,13 +240,14 @@ function main() {
     console.log('| Document | Words | Raw score | Exempt score | Budget |');
     console.log('|---|---:|---:|---:|---:|');
     for (const r of rows) {
-      console.log(`| \`${r.file}\` | ${r.words.toLocaleString()} | ${r.rawScore} | **${r.exemptScore}** | ${r.budget} |`);
+      const exemptCell = r.declined ? 'declined' : `**${r.exemptScore}**`;
+      console.log(`| \`${r.file}\` | ${r.words.toLocaleString()} | ${r.rawScore} | ${exemptCell} | ${r.budget} |`);
     }
   } else {
     console.log('\nself-scan — this skill\'s detector against this skill\'s docs\n');
     console.log('  file                      words    raw  exempt  budget');
     for (const r of rows) {
-      const flag = r.overBudget ? '  OVER' : '';
+      const flag = r.declined ? '  DECLINED' : (r.overBudget ? '  OVER' : '');
       console.log(
         `  ${r.file.padEnd(24)}${String(r.words).padStart(6)}${String(r.rawScore).padStart(7)}${String(r.exemptScore).padStart(8)}${String(r.budget).padStart(8)}${flag}`,
       );
@@ -246,6 +265,11 @@ function main() {
   }
 
   if (args.includes('--check')) {
+    const declined = rows.filter((r) => r.declined);
+    if (declined.length) {
+      console.error(`\nFAIL — ${declined.length} file(s) could not be scored (unsupported script): ${declined.map((r) => r.file).join(', ')}`);
+      process.exit(1);
+    }
     const over = rows.filter((r) => r.overBudget);
     if (over.length) {
       console.error(`\nFAIL — ${over.length} file(s) over budget: ${over.map((r) => r.file).join(', ')}`);

skills/ai-writing-detector/scripts/patterns.js

@@ -1740,6 +1740,27 @@ const AIDetector = (() => {
     sourceMap = norm.sourceMap;
 
     const wordCount = countWords(text);
+    // Unsegmented-script check (GH-241): Chinese and Japanese carry no
+    // inter-word spaces, so word segmentation cannot measure them — a long
+    // document counts as one \S+ run and would misreport as "Too short",
+    // while newline-wrapped lines each count as a word and would score
+    // without segmentation. The check therefore runs before the word gate
+    // and declines only when CJK characters dominate the non-whitespace
+    // text, so short English documents with an incidental place name or
+    // single Han character stay scorable. Han, Hiragana, and Katakana
+    // ranges (including halfwidth) signal an unsegmented script; Hangul is
+    // space-separated and segments fine, so it is excluded.
+    const cjkChars = (text.match(/[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff66-\uff9d]/g) || []).length;
+    if (cjkChars > 0 && cjkChars * 2 >= (text.match(/\S/g) || []).length) {
+      return {
+        ...buildV2Defaults('UNSCORED', 'low'),
+        score: 0,
+        label: 'Unsupported script',
+        issues: [],
+        stats: { wordCount, cjkChars, reason: 'unsegmented-script document: no inter-word spaces to count', contextMode, contextModeFallback, sourceMode, sourceModeFallback, maskedFrontmatter, maskedHtmlComments },
+        unsupportedScript: true,
+      };
+    }
     if (wordCount < 10) {
       return {
         ...buildV2Defaults('UNSCORED', 'low'),

skills/avoid-ai-writing/detector/patterns.js

@@ -1740,6 +1740,27 @@ const AIDetector = (() => {
     sourceMap = norm.sourceMap;
 
     const wordCount = countWords(text);
+    // Unsegmented-script check (GH-241): Chinese and Japanese carry no
+    // inter-word spaces, so word segmentation cannot measure them — a long
+    // document counts as one \S+ run and would misreport as "Too short",
+    // while newline-wrapped lines each count as a word and would score
+    // without segmentation. The check therefore runs before the word gate
+    // and declines only when CJK characters dominate the non-whitespace
+    // text, so short English documents with an incidental place name or
+    // single Han character stay scorable. Han, Hiragana, and Katakana
+    // ranges (including halfwidth) signal an unsegmented script; Hangul is
+    // space-separated and segments fine, so it is excluded.
+    const cjkChars = (text.match(/[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff66-\uff9d]/g) || []).length;
+    if (cjkChars > 0 && cjkChars * 2 >= (text.match(/\S/g) || []).length) {
+      return {
+        ...buildV2Defaults('UNSCORED', 'low'),
+        score: 0,
+        label: 'Unsupported script',
+        issues: [],
+        stats: { wordCount, cjkChars, reason: 'unsegmented-script document: no inter-word spaces to count', contextMode, contextModeFallback, sourceMode, sourceModeFallback, maskedFrontmatter, maskedHtmlComments },
+        unsupportedScript: true,
+      };
+    }
     if (wordCount < 10) {
       return {
         ...buildV2Defaults('UNSCORED', 'low'),

skills/preservation-verifier/scripts/patterns.js

@@ -1740,6 +1740,27 @@ const AIDetector = (() => {
     sourceMap = norm.sourceMap;
 
     const wordCount = countWords(text);
+    // Unsegmented-script check (GH-241): Chinese and Japanese carry no
+    // inter-word spaces, so word segmentation cannot measure them — a long
+    // document counts as one \S+ run and would misreport as "Too short",
+    // while newline-wrapped lines each count as a word and would score
+    // without segmentation. The check therefore runs before the word gate
+    // and declines only when CJK characters dominate the non-whitespace
+    // text, so short English documents with an incidental place name or
+    // single Han character stay scorable. Han, Hiragana, and Katakana
+    // ranges (including halfwidth) signal an unsegmented script; Hangul is
+    // space-separated and segments fine, so it is excluded.
+    const cjkChars = (text.match(/[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff66-\uff9d]/g) || []).length;
+    if (cjkChars > 0 && cjkChars * 2 >= (text.match(/\S/g) || []).length) {
+      return {
+        ...buildV2Defaults('UNSCORED', 'low'),
+        score: 0,
+        label: 'Unsupported script',
+        issues: [],
+        stats: { wordCount, cjkChars, reason: 'unsegmented-script document: no inter-word spaces to count', contextMode, contextModeFallback, sourceMode, sourceModeFallback, maskedFrontmatter, maskedHtmlComments },
+        unsupportedScript: true,
+      };
+    }
     if (wordCount < 10) {
       return {
         ...buildV2Defaults('UNSCORED', 'low'),