export interface KeywordCandidate { id: string start: number end: number text: string score: number selected: boolean } export interface MosaicOptions { volume: number disorder: number seed: number } export interface MosaicResult { output: string inputLength: number outputLength: number retainedLength: number } interface OutputPart { type: 'hidden' | 'retained' | 'newline' text: string originalLength: number noiseLength: number weight: number } type Random = () => number const STOP_WORDS = new Set([ '一个', '一些', '一种', '不是', '不能', '不过', '为了', '之后', '之前', '于是', '他们', '但是', '以及', '仍然', '什么', '从而', '以后', '已经', '以为', '每一', '你们', '使得', '关于', '其实', '再也', '只是', '可以', '可能', '后来', '因为', '如果', '它们', '对于', '就是', '并且', '开始', '我们', '所以', '所有', '时候', '是否', '然后', '现在', '由于', '直到', '自己', '都在', '虽然', '这个', '这些', '这种', '那个', '那些', '那么', '里面', '重新', '非常', '没有', '正在', '能够', '还是', '进行', '通过', '需要', '一起', '一样', '一般', '一点', '一次', '一直', '一切', '不会', '不要', '不再', '不只', '不但', '不管', '不论', '而且', '或者', '其中', '其他', '某个', '某些', '这里', '那里', '怎样', '如何', '为何', '何时', '我', '你', '他', '她', '它', '这', '那', '是', '的', '地', '得', '了', '着', '过', '在', '和', '与', '及', '而', '也', '都', '就', '又', '还', '才', '次', '被', '把', '将', '会', '能', '可', '不', '没', '很', '更', '最', '让', '从', '到', '对', '于', '以', '为', '所', '其', ]) const SIMPLE_HAN = [ '丌', '乂', '乜', '亍', '兀', '仝', '冇', '夬', '尢', '孑', '孓', '廿', '卅', '卌', '凼', '彳', '氕', '氘', '叵', '玊', '旡', '乇', '亓', '伋', '佀', '佧', '侘', ] as const const GLITCH_SYMBOLS = [ '▒', '▓', '░', '╳', '⍉', '⌁', '⊘', '⊗', '⊙', '∷', '※', '⌗', '⎔', '␀', '␟', '�', ] as const const FOREIGN_GLYPHS = [ 'Ж', 'Д', 'æ', 'Ø', 'λ', 'ϟ', '҂', 'ヰ', 'ヱ', 'ヲ', 'ヵ', 'メ', ] as const const ENCODING_FRAGMENTS = ['Ã', 'Â', 'æ', 'å', '¤', '锟', '斤', '拷'] as const const COMPLEX_HAN = ['黻', '鬯', '夔', '爨', '龘', '靐', '罅', '黹'] as const const HAN_PATTERN = /\p{Script=Han}/u const WORD_PATTERN = /[\p{L}\p{N}]/u const wordSegmenter = new Intl.Segmenter('zh-CN', { granularity: 'word' }) const graphemeSegmenter = new Intl.Segmenter('zh-CN', { granularity: 'grapheme', }) function createRandom(seed: number): Random { let state = seed >>> 0 return () => { state += 0x6d2b79f5 let value = state value = Math.imul(value ^ (value >>> 15), value | 1) value ^= value + Math.imul(value ^ (value >>> 7), value | 61) return ((value ^ (value >>> 14)) >>> 0) / 4294967296 } } function hashText(value: string): number { let hash = 2166136261 for (const character of value) { hash ^= character.codePointAt(0) ?? 0 hash = Math.imul(hash, 16777619) } return hash >>> 0 } function countGraphemes(value: string): number { let count = 0 for (const segment of graphemeSegmenter.segment(value)) { if (segment.segment !== '\n' && segment.segment !== '\r') { count += 1 } } return count } function isKeywordCandidate(segment: string, isWordLike: boolean): boolean { if (!isWordLike || STOP_WORDS.has(segment) || !WORD_PATTERN.test(segment)) { return false } const length = countGraphemes(segment) if (HAN_PATTERN.test(segment)) { return length >= 1 && length <= 8 } return length >= 2 && length <= 16 } function scoreCandidate( text: string, start: number, frequency: number, selectionSeed: number, ): number { const length = countGraphemes(text) const lengthScore = length === 2 ? 2.8 : length === 3 ? 2.4 : length === 4 ? 1.8 : 0.7 const frequencyScore = Math.min(frequency - 1, 3) * 1.2 const hanScore = HAN_PATTERN.test(text) ? 0.8 : 0.3 const random = createRandom(hashText(`${text}:${start}:${selectionSeed}`))() return lengthScore + frequencyScore + hanScore + random * 1.4 } export function selectKeywords( text: string, retention: number, selectionSeed = 1, ): KeywordCandidate[] { const segments = Array.from(wordSegmenter.segment(text)) const candidateSegments = segments.filter((segment) => isKeywordCandidate(segment.segment, Boolean(segment.isWordLike)), ) const frequencies = new Map() for (const segment of candidateSegments) { frequencies.set(segment.segment, (frequencies.get(segment.segment) ?? 0) + 1) } const candidates = candidateSegments.map((segment) => ({ id: `${segment.index}:${segment.index + segment.segment.length}`, start: segment.index, end: segment.index + segment.segment.length, text: segment.segment, score: scoreCandidate( segment.segment, segment.index, frequencies.get(segment.segment) ?? 1, selectionSeed, ), selected: false, })) if (candidates.length === 0) { return candidates } const desiredCount = retention <= 0 ? 0 : Math.min( candidates.length, Math.max(1, Math.ceil(candidates.length * (retention / 100))), ) const ranked = [...candidates].sort( (left, right) => right.score - left.score || left.start - right.start, ) const selectedIds = new Set( ranked.slice(0, desiredCount).map((candidate) => candidate.id), ) return candidates.map((candidate) => ({ ...candidate, selected: selectedIds.has(candidate.id), })) } function choose(values: readonly T[], random: Random): T { return values[Math.floor(random() * values.length)] } function chooseNoiseGlyph(random: Random): string { const categoryRoll = random() * 100 if (categoryRoll < 60) { return choose(SIMPLE_HAN, random) } if (categoryRoll < 80) { return choose(GLITCH_SYMBOLS, random) } if (categoryRoll < 90) { return choose(FOREIGN_GLYPHS, random) } if (categoryRoll < 97) { return choose(ENCODING_FRAGMENTS, random) } return choose(COMPLEX_HAN, random) } function generateNoise(length: number, disorder: number, random: Random): string { let output = '' let previous = '' const repeatChance = 0.02 + disorder * 0.0015 for (let index = 0; index < length; index += 1) { const shouldRepeat = previous !== '' && random() < repeatChance const glyph = shouldRepeat ? previous : chooseNoiseGlyph(random) output += glyph previous = glyph } return output } function pushHiddenParts(parts: OutputPart[], text: string): void { const lines = text.split(/(\r?\n)/) for (const line of lines) { if (line === '\n' || line === '\r\n') { parts.push({ type: 'newline', text: '\n', originalLength: 0, noiseLength: 0, weight: 0, }) continue } const length = countGraphemes(line) if (length > 0) { parts.push({ type: 'hidden', text: line, originalLength: length, noiseLength: 0, weight: 0, }) } } } function buildOutputParts( text: string, retainedRanges: KeywordCandidate[], ): OutputPart[] { const parts: OutputPart[] = [] let cursor = 0 for (const range of retainedRanges) { if (range.start > cursor) { pushHiddenParts(parts, text.slice(cursor, range.start)) } parts.push({ type: 'retained', text: text.slice(range.start, range.end), originalLength: countGraphemes(text.slice(range.start, range.end)), noiseLength: 0, weight: 0, }) cursor = range.end } if (cursor < text.length) { pushHiddenParts(parts, text.slice(cursor)) } return parts } function distributeNoiseBudget( parts: OutputPart[], budget: number, disorder: number, random: Random, ): void { const hiddenParts = parts.filter( (part) => part.type === 'hidden' && part.originalLength > 0, ) if (hiddenParts.length === 0 || budget <= 0) { return } const baseAllocation = budget >= hiddenParts.length ? 1 : 0 let remaining = budget - baseAllocation * hiddenParts.length const variability = disorder / 100 for (const part of hiddenParts) { const localScale = 1 + (random() * 2 - 1) * variability * 0.62 part.weight = Math.max(0.05, part.originalLength * localScale) part.noiseLength = baseAllocation } if (remaining <= 0) { for (let index = 0; index < budget; index += 1) { hiddenParts[index].noiseLength = 1 } return } const totalWeight = hiddenParts.reduce((sum, part) => sum + part.weight, 0) const allocations = hiddenParts.map((part) => { const exact = (part.weight / totalWeight) * remaining const whole = Math.floor(exact) part.noiseLength += whole return { part, fraction: exact - whole } }) const allocated = allocations.reduce( (sum, allocation) => sum + Math.floor((allocation.part.weight / totalWeight) * remaining), 0, ) remaining -= allocated allocations.sort((left, right) => right.fraction - left.fraction) for (let index = 0; index < remaining; index += 1) { allocations[index % allocations.length].part.noiseLength += 1 } } export function generateMosaic( text: string, candidates: KeywordCandidate[], options: MosaicOptions, ): MosaicResult { const random = createRandom(options.seed ^ hashText(text)) const retainedRanges = candidates .filter((candidate) => candidate.selected) .sort((left, right) => left.start - right.start) const parts = buildOutputParts(text, retainedRanges) const inputLength = countGraphemes(text) const retainedLength = parts .filter((part) => part.type === 'retained') .reduce((sum, part) => sum + part.originalLength, 0) const jitter = (random() * 2 - 1) * (options.disorder / 100) * 0.035 const targetLength = Math.max( retainedLength, Math.round(inputLength * (options.volume / 100 + jitter)), ) const noiseBudget = Math.max(0, targetLength - retainedLength) distributeNoiseBudget(parts, noiseBudget, options.disorder, random) const output = parts .map((part) => { if (part.type === 'hidden') { return generateNoise(part.noiseLength, options.disorder, random) } return part.text }) .join('') return { output, inputLength, outputLength: countGraphemes(output), retainedLength, } }