557 lines
11 KiB
TypeScript
557 lines
11 KiB
TypeScript
export interface KeywordCandidate {
|
||
id: string
|
||
start: number
|
||
end: number
|
||
text: string
|
||
score: number
|
||
selected: boolean
|
||
}
|
||
|
||
export interface MosaicOptions {
|
||
volume: number
|
||
disorder: number
|
||
seed: number
|
||
}
|
||
|
||
export interface MosaicResult {
|
||
output: string
|
||
inputLength: number
|
||
outputLength: number
|
||
retainedLength: number
|
||
}
|
||
|
||
interface OutputPart {
|
||
type: 'hidden' | 'retained' | 'newline'
|
||
text: string
|
||
originalLength: number
|
||
noiseLength: number
|
||
weight: number
|
||
}
|
||
|
||
type Random = () => number
|
||
|
||
const STOP_WORDS = new Set([
|
||
'一个',
|
||
'一些',
|
||
'一种',
|
||
'不是',
|
||
'不能',
|
||
'不过',
|
||
'为了',
|
||
'之后',
|
||
'之前',
|
||
'于是',
|
||
'他们',
|
||
'但是',
|
||
'以及',
|
||
'仍然',
|
||
'什么',
|
||
'从而',
|
||
'以后',
|
||
'已经',
|
||
'以为',
|
||
'每一',
|
||
'你们',
|
||
'使得',
|
||
'关于',
|
||
'其实',
|
||
'再也',
|
||
'只是',
|
||
'可以',
|
||
'可能',
|
||
'后来',
|
||
'因为',
|
||
'如果',
|
||
'它们',
|
||
'对于',
|
||
'就是',
|
||
'并且',
|
||
'开始',
|
||
'我们',
|
||
'所以',
|
||
'所有',
|
||
'时候',
|
||
'是否',
|
||
'然后',
|
||
'现在',
|
||
'由于',
|
||
'直到',
|
||
'自己',
|
||
'都在',
|
||
'虽然',
|
||
'这个',
|
||
'这些',
|
||
'这种',
|
||
'那个',
|
||
'那些',
|
||
'那么',
|
||
'里面',
|
||
'重新',
|
||
'非常',
|
||
'没有',
|
||
'正在',
|
||
'能够',
|
||
'还是',
|
||
'进行',
|
||
'通过',
|
||
'需要',
|
||
'一起',
|
||
'一样',
|
||
'一般',
|
||
'一点',
|
||
'一次',
|
||
'一直',
|
||
'一切',
|
||
'不会',
|
||
'不要',
|
||
'不再',
|
||
'不只',
|
||
'不但',
|
||
'不管',
|
||
'不论',
|
||
'而且',
|
||
'或者',
|
||
'其中',
|
||
'其他',
|
||
'某个',
|
||
'某些',
|
||
'这里',
|
||
'那里',
|
||
'怎样',
|
||
'如何',
|
||
'为何',
|
||
'何时',
|
||
'我',
|
||
'你',
|
||
'他',
|
||
'她',
|
||
'它',
|
||
'这',
|
||
'那',
|
||
'是',
|
||
'的',
|
||
'地',
|
||
'得',
|
||
'了',
|
||
'着',
|
||
'过',
|
||
'在',
|
||
'和',
|
||
'与',
|
||
'及',
|
||
'而',
|
||
'也',
|
||
'都',
|
||
'就',
|
||
'又',
|
||
'还',
|
||
'才',
|
||
'次',
|
||
'被',
|
||
'把',
|
||
'将',
|
||
'会',
|
||
'能',
|
||
'可',
|
||
'不',
|
||
'没',
|
||
'很',
|
||
'更',
|
||
'最',
|
||
'让',
|
||
'从',
|
||
'到',
|
||
'对',
|
||
'于',
|
||
'以',
|
||
'为',
|
||
'所',
|
||
'其',
|
||
])
|
||
|
||
const SIMPLE_HAN = [
|
||
'丌',
|
||
'乂',
|
||
'乜',
|
||
'亍',
|
||
'兀',
|
||
'仝',
|
||
'冇',
|
||
'夬',
|
||
'尢',
|
||
'孑',
|
||
'孓',
|
||
'廿',
|
||
'卅',
|
||
'卌',
|
||
'凼',
|
||
'彳',
|
||
'氕',
|
||
'氘',
|
||
'叵',
|
||
'玊',
|
||
'旡',
|
||
'乇',
|
||
'亓',
|
||
'伋',
|
||
'佀',
|
||
'佧',
|
||
'侘',
|
||
] as const
|
||
|
||
const GLITCH_SYMBOLS = [
|
||
'▒',
|
||
'▓',
|
||
'░',
|
||
'╳',
|
||
'⍉',
|
||
'⌁',
|
||
'⊘',
|
||
'⊗',
|
||
'⊙',
|
||
'∷',
|
||
'※',
|
||
'⌗',
|
||
'⎔',
|
||
'␀',
|
||
'␟',
|
||
'�',
|
||
] as const
|
||
|
||
const FOREIGN_GLYPHS = [
|
||
'Ж',
|
||
'Д',
|
||
'æ',
|
||
'Ø',
|
||
'λ',
|
||
'ϟ',
|
||
'҂',
|
||
'ヰ',
|
||
'ヱ',
|
||
'ヲ',
|
||
'ヵ',
|
||
'ﾒ',
|
||
] as const
|
||
|
||
const ENCODING_FRAGMENTS = ['Ã', 'Â', 'æ', 'å', '¤', '锟', '斤', '拷'] as const
|
||
|
||
const COMPLEX_HAN = ['黻', '鬯', '夔', '爨', '龘', '靐', '罅', '黹'] as const
|
||
|
||
const HAN_PATTERN = /\p{Script=Han}/u
|
||
const WORD_PATTERN = /[\p{L}\p{N}]/u
|
||
|
||
const wordSegmenter = new Intl.Segmenter('zh-CN', { granularity: 'word' })
|
||
const graphemeSegmenter = new Intl.Segmenter('zh-CN', {
|
||
granularity: 'grapheme',
|
||
})
|
||
|
||
function createRandom(seed: number): Random {
|
||
let state = seed >>> 0
|
||
|
||
return () => {
|
||
state += 0x6d2b79f5
|
||
let value = state
|
||
value = Math.imul(value ^ (value >>> 15), value | 1)
|
||
value ^= value + Math.imul(value ^ (value >>> 7), value | 61)
|
||
return ((value ^ (value >>> 14)) >>> 0) / 4294967296
|
||
}
|
||
}
|
||
|
||
function hashText(value: string): number {
|
||
let hash = 2166136261
|
||
|
||
for (const character of value) {
|
||
hash ^= character.codePointAt(0) ?? 0
|
||
hash = Math.imul(hash, 16777619)
|
||
}
|
||
|
||
return hash >>> 0
|
||
}
|
||
|
||
function countGraphemes(value: string): number {
|
||
let count = 0
|
||
|
||
for (const segment of graphemeSegmenter.segment(value)) {
|
||
if (segment.segment !== '\n' && segment.segment !== '\r') {
|
||
count += 1
|
||
}
|
||
}
|
||
|
||
return count
|
||
}
|
||
|
||
function isKeywordCandidate(segment: string, isWordLike: boolean): boolean {
|
||
if (!isWordLike || STOP_WORDS.has(segment) || !WORD_PATTERN.test(segment)) {
|
||
return false
|
||
}
|
||
|
||
const length = countGraphemes(segment)
|
||
|
||
if (HAN_PATTERN.test(segment)) {
|
||
return length >= 1 && length <= 8
|
||
}
|
||
|
||
return length >= 2 && length <= 16
|
||
}
|
||
|
||
function scoreCandidate(
|
||
text: string,
|
||
start: number,
|
||
frequency: number,
|
||
selectionSeed: number,
|
||
): number {
|
||
const length = countGraphemes(text)
|
||
const lengthScore =
|
||
length === 2 ? 2.8 : length === 3 ? 2.4 : length === 4 ? 1.8 : 0.7
|
||
const frequencyScore = Math.min(frequency - 1, 3) * 1.2
|
||
const hanScore = HAN_PATTERN.test(text) ? 0.8 : 0.3
|
||
const random = createRandom(hashText(`${text}:${start}:${selectionSeed}`))()
|
||
|
||
return lengthScore + frequencyScore + hanScore + random * 1.4
|
||
}
|
||
|
||
export function selectKeywords(
|
||
text: string,
|
||
retention: number,
|
||
selectionSeed = 1,
|
||
): KeywordCandidate[] {
|
||
const segments = Array.from(wordSegmenter.segment(text))
|
||
const candidateSegments = segments.filter((segment) =>
|
||
isKeywordCandidate(segment.segment, Boolean(segment.isWordLike)),
|
||
)
|
||
const frequencies = new Map<string, number>()
|
||
|
||
for (const segment of candidateSegments) {
|
||
frequencies.set(segment.segment, (frequencies.get(segment.segment) ?? 0) + 1)
|
||
}
|
||
|
||
const candidates = candidateSegments.map((segment) => ({
|
||
id: `${segment.index}:${segment.index + segment.segment.length}`,
|
||
start: segment.index,
|
||
end: segment.index + segment.segment.length,
|
||
text: segment.segment,
|
||
score: scoreCandidate(
|
||
segment.segment,
|
||
segment.index,
|
||
frequencies.get(segment.segment) ?? 1,
|
||
selectionSeed,
|
||
),
|
||
selected: false,
|
||
}))
|
||
|
||
if (candidates.length === 0) {
|
||
return candidates
|
||
}
|
||
|
||
const desiredCount =
|
||
retention <= 0
|
||
? 0
|
||
: Math.min(
|
||
candidates.length,
|
||
Math.max(1, Math.ceil(candidates.length * (retention / 100))),
|
||
)
|
||
const ranked = [...candidates].sort(
|
||
(left, right) => right.score - left.score || left.start - right.start,
|
||
)
|
||
const selectedIds = new Set(
|
||
ranked.slice(0, desiredCount).map((candidate) => candidate.id),
|
||
)
|
||
|
||
return candidates.map((candidate) => ({
|
||
...candidate,
|
||
selected: selectedIds.has(candidate.id),
|
||
}))
|
||
}
|
||
|
||
function choose<T>(values: readonly T[], random: Random): T {
|
||
return values[Math.floor(random() * values.length)]
|
||
}
|
||
|
||
function chooseNoiseGlyph(random: Random): string {
|
||
const categoryRoll = random() * 100
|
||
|
||
if (categoryRoll < 60) {
|
||
return choose(SIMPLE_HAN, random)
|
||
}
|
||
|
||
if (categoryRoll < 80) {
|
||
return choose(GLITCH_SYMBOLS, random)
|
||
}
|
||
|
||
if (categoryRoll < 90) {
|
||
return choose(FOREIGN_GLYPHS, random)
|
||
}
|
||
|
||
if (categoryRoll < 97) {
|
||
return choose(ENCODING_FRAGMENTS, random)
|
||
}
|
||
|
||
return choose(COMPLEX_HAN, random)
|
||
}
|
||
|
||
function generateNoise(length: number, disorder: number, random: Random): string {
|
||
let output = ''
|
||
let previous = ''
|
||
const repeatChance = 0.02 + disorder * 0.0015
|
||
|
||
for (let index = 0; index < length; index += 1) {
|
||
const shouldRepeat = previous !== '' && random() < repeatChance
|
||
const glyph = shouldRepeat ? previous : chooseNoiseGlyph(random)
|
||
|
||
output += glyph
|
||
previous = glyph
|
||
}
|
||
|
||
return output
|
||
}
|
||
|
||
function pushHiddenParts(parts: OutputPart[], text: string): void {
|
||
const lines = text.split(/(\r?\n)/)
|
||
|
||
for (const line of lines) {
|
||
if (line === '\n' || line === '\r\n') {
|
||
parts.push({
|
||
type: 'newline',
|
||
text: '\n',
|
||
originalLength: 0,
|
||
noiseLength: 0,
|
||
weight: 0,
|
||
})
|
||
continue
|
||
}
|
||
|
||
const length = countGraphemes(line)
|
||
|
||
if (length > 0) {
|
||
parts.push({
|
||
type: 'hidden',
|
||
text: line,
|
||
originalLength: length,
|
||
noiseLength: 0,
|
||
weight: 0,
|
||
})
|
||
}
|
||
}
|
||
}
|
||
|
||
function buildOutputParts(
|
||
text: string,
|
||
retainedRanges: KeywordCandidate[],
|
||
): OutputPart[] {
|
||
const parts: OutputPart[] = []
|
||
let cursor = 0
|
||
|
||
for (const range of retainedRanges) {
|
||
if (range.start > cursor) {
|
||
pushHiddenParts(parts, text.slice(cursor, range.start))
|
||
}
|
||
|
||
parts.push({
|
||
type: 'retained',
|
||
text: text.slice(range.start, range.end),
|
||
originalLength: countGraphemes(text.slice(range.start, range.end)),
|
||
noiseLength: 0,
|
||
weight: 0,
|
||
})
|
||
cursor = range.end
|
||
}
|
||
|
||
if (cursor < text.length) {
|
||
pushHiddenParts(parts, text.slice(cursor))
|
||
}
|
||
|
||
return parts
|
||
}
|
||
|
||
function distributeNoiseBudget(
|
||
parts: OutputPart[],
|
||
budget: number,
|
||
disorder: number,
|
||
random: Random,
|
||
): void {
|
||
const hiddenParts = parts.filter(
|
||
(part) => part.type === 'hidden' && part.originalLength > 0,
|
||
)
|
||
|
||
if (hiddenParts.length === 0 || budget <= 0) {
|
||
return
|
||
}
|
||
|
||
const baseAllocation = budget >= hiddenParts.length ? 1 : 0
|
||
let remaining = budget - baseAllocation * hiddenParts.length
|
||
const variability = disorder / 100
|
||
|
||
for (const part of hiddenParts) {
|
||
const localScale = 1 + (random() * 2 - 1) * variability * 0.62
|
||
part.weight = Math.max(0.05, part.originalLength * localScale)
|
||
part.noiseLength = baseAllocation
|
||
}
|
||
|
||
if (remaining <= 0) {
|
||
for (let index = 0; index < budget; index += 1) {
|
||
hiddenParts[index].noiseLength = 1
|
||
}
|
||
return
|
||
}
|
||
|
||
const totalWeight = hiddenParts.reduce((sum, part) => sum + part.weight, 0)
|
||
const allocations = hiddenParts.map((part) => {
|
||
const exact = (part.weight / totalWeight) * remaining
|
||
const whole = Math.floor(exact)
|
||
part.noiseLength += whole
|
||
return { part, fraction: exact - whole }
|
||
})
|
||
const allocated = allocations.reduce(
|
||
(sum, allocation) => sum + Math.floor((allocation.part.weight / totalWeight) * remaining),
|
||
0,
|
||
)
|
||
|
||
remaining -= allocated
|
||
allocations.sort((left, right) => right.fraction - left.fraction)
|
||
|
||
for (let index = 0; index < remaining; index += 1) {
|
||
allocations[index % allocations.length].part.noiseLength += 1
|
||
}
|
||
}
|
||
|
||
export function generateMosaic(
|
||
text: string,
|
||
candidates: KeywordCandidate[],
|
||
options: MosaicOptions,
|
||
): MosaicResult {
|
||
const random = createRandom(options.seed ^ hashText(text))
|
||
const retainedRanges = candidates
|
||
.filter((candidate) => candidate.selected)
|
||
.sort((left, right) => left.start - right.start)
|
||
const parts = buildOutputParts(text, retainedRanges)
|
||
const inputLength = countGraphemes(text)
|
||
const retainedLength = parts
|
||
.filter((part) => part.type === 'retained')
|
||
.reduce((sum, part) => sum + part.originalLength, 0)
|
||
const jitter = (random() * 2 - 1) * (options.disorder / 100) * 0.035
|
||
const targetLength = Math.max(
|
||
retainedLength,
|
||
Math.round(inputLength * (options.volume / 100 + jitter)),
|
||
)
|
||
const noiseBudget = Math.max(0, targetLength - retainedLength)
|
||
|
||
distributeNoiseBudget(parts, noiseBudget, options.disorder, random)
|
||
|
||
const output = parts
|
||
.map((part) => {
|
||
if (part.type === 'hidden') {
|
||
return generateNoise(part.noiseLength, options.disorder, random)
|
||
}
|
||
|
||
return part.text
|
||
})
|
||
.join('')
|
||
|
||
return {
|
||
output,
|
||
inputLength,
|
||
outputLength: countGraphemes(output),
|
||
retainedLength,
|
||
}
|
||
}
|