recreate project
This commit is contained in:
+556
@@ -0,0 +1,556 @@
|
||||
export interface KeywordCandidate {
|
||||
id: string
|
||||
start: number
|
||||
end: number
|
||||
text: string
|
||||
score: number
|
||||
selected: boolean
|
||||
}
|
||||
|
||||
export interface MosaicOptions {
|
||||
volume: number
|
||||
disorder: number
|
||||
seed: number
|
||||
}
|
||||
|
||||
export interface MosaicResult {
|
||||
output: string
|
||||
inputLength: number
|
||||
outputLength: number
|
||||
retainedLength: number
|
||||
}
|
||||
|
||||
interface OutputPart {
|
||||
type: 'hidden' | 'retained' | 'newline'
|
||||
text: string
|
||||
originalLength: number
|
||||
noiseLength: number
|
||||
weight: number
|
||||
}
|
||||
|
||||
type Random = () => number
|
||||
|
||||
const STOP_WORDS = new Set([
|
||||
'一个',
|
||||
'一些',
|
||||
'一种',
|
||||
'不是',
|
||||
'不能',
|
||||
'不过',
|
||||
'为了',
|
||||
'之后',
|
||||
'之前',
|
||||
'于是',
|
||||
'他们',
|
||||
'但是',
|
||||
'以及',
|
||||
'仍然',
|
||||
'什么',
|
||||
'从而',
|
||||
'以后',
|
||||
'已经',
|
||||
'以为',
|
||||
'每一',
|
||||
'你们',
|
||||
'使得',
|
||||
'关于',
|
||||
'其实',
|
||||
'再也',
|
||||
'只是',
|
||||
'可以',
|
||||
'可能',
|
||||
'后来',
|
||||
'因为',
|
||||
'如果',
|
||||
'它们',
|
||||
'对于',
|
||||
'就是',
|
||||
'并且',
|
||||
'开始',
|
||||
'我们',
|
||||
'所以',
|
||||
'所有',
|
||||
'时候',
|
||||
'是否',
|
||||
'然后',
|
||||
'现在',
|
||||
'由于',
|
||||
'直到',
|
||||
'自己',
|
||||
'都在',
|
||||
'虽然',
|
||||
'这个',
|
||||
'这些',
|
||||
'这种',
|
||||
'那个',
|
||||
'那些',
|
||||
'那么',
|
||||
'里面',
|
||||
'重新',
|
||||
'非常',
|
||||
'没有',
|
||||
'正在',
|
||||
'能够',
|
||||
'还是',
|
||||
'进行',
|
||||
'通过',
|
||||
'需要',
|
||||
'一起',
|
||||
'一样',
|
||||
'一般',
|
||||
'一点',
|
||||
'一次',
|
||||
'一直',
|
||||
'一切',
|
||||
'不会',
|
||||
'不要',
|
||||
'不再',
|
||||
'不只',
|
||||
'不但',
|
||||
'不管',
|
||||
'不论',
|
||||
'而且',
|
||||
'或者',
|
||||
'其中',
|
||||
'其他',
|
||||
'某个',
|
||||
'某些',
|
||||
'这里',
|
||||
'那里',
|
||||
'怎样',
|
||||
'如何',
|
||||
'为何',
|
||||
'何时',
|
||||
'我',
|
||||
'你',
|
||||
'他',
|
||||
'她',
|
||||
'它',
|
||||
'这',
|
||||
'那',
|
||||
'是',
|
||||
'的',
|
||||
'地',
|
||||
'得',
|
||||
'了',
|
||||
'着',
|
||||
'过',
|
||||
'在',
|
||||
'和',
|
||||
'与',
|
||||
'及',
|
||||
'而',
|
||||
'也',
|
||||
'都',
|
||||
'就',
|
||||
'又',
|
||||
'还',
|
||||
'才',
|
||||
'次',
|
||||
'被',
|
||||
'把',
|
||||
'将',
|
||||
'会',
|
||||
'能',
|
||||
'可',
|
||||
'不',
|
||||
'没',
|
||||
'很',
|
||||
'更',
|
||||
'最',
|
||||
'让',
|
||||
'从',
|
||||
'到',
|
||||
'对',
|
||||
'于',
|
||||
'以',
|
||||
'为',
|
||||
'所',
|
||||
'其',
|
||||
])
|
||||
|
||||
const SIMPLE_HAN = [
|
||||
'丌',
|
||||
'乂',
|
||||
'乜',
|
||||
'亍',
|
||||
'兀',
|
||||
'仝',
|
||||
'冇',
|
||||
'夬',
|
||||
'尢',
|
||||
'孑',
|
||||
'孓',
|
||||
'廿',
|
||||
'卅',
|
||||
'卌',
|
||||
'凼',
|
||||
'彳',
|
||||
'氕',
|
||||
'氘',
|
||||
'叵',
|
||||
'玊',
|
||||
'旡',
|
||||
'乇',
|
||||
'亓',
|
||||
'伋',
|
||||
'佀',
|
||||
'佧',
|
||||
'侘',
|
||||
] as const
|
||||
|
||||
const GLITCH_SYMBOLS = [
|
||||
'▒',
|
||||
'▓',
|
||||
'░',
|
||||
'╳',
|
||||
'⍉',
|
||||
'⌁',
|
||||
'⊘',
|
||||
'⊗',
|
||||
'⊙',
|
||||
'∷',
|
||||
'※',
|
||||
'⌗',
|
||||
'⎔',
|
||||
'␀',
|
||||
'␟',
|
||||
'�',
|
||||
] as const
|
||||
|
||||
const FOREIGN_GLYPHS = [
|
||||
'Ж',
|
||||
'Д',
|
||||
'æ',
|
||||
'Ø',
|
||||
'λ',
|
||||
'ϟ',
|
||||
'҂',
|
||||
'ヰ',
|
||||
'ヱ',
|
||||
'ヲ',
|
||||
'ヵ',
|
||||
'ﾒ',
|
||||
] as const
|
||||
|
||||
const ENCODING_FRAGMENTS = ['Ã', 'Â', 'æ', 'å', '¤', '锟', '斤', '拷'] as const
|
||||
|
||||
const COMPLEX_HAN = ['黻', '鬯', '夔', '爨', '龘', '靐', '罅', '黹'] as const
|
||||
|
||||
const HAN_PATTERN = /\p{Script=Han}/u
|
||||
const WORD_PATTERN = /[\p{L}\p{N}]/u
|
||||
|
||||
const wordSegmenter = new Intl.Segmenter('zh-CN', { granularity: 'word' })
|
||||
const graphemeSegmenter = new Intl.Segmenter('zh-CN', {
|
||||
granularity: 'grapheme',
|
||||
})
|
||||
|
||||
function createRandom(seed: number): Random {
|
||||
let state = seed >>> 0
|
||||
|
||||
return () => {
|
||||
state += 0x6d2b79f5
|
||||
let value = state
|
||||
value = Math.imul(value ^ (value >>> 15), value | 1)
|
||||
value ^= value + Math.imul(value ^ (value >>> 7), value | 61)
|
||||
return ((value ^ (value >>> 14)) >>> 0) / 4294967296
|
||||
}
|
||||
}
|
||||
|
||||
function hashText(value: string): number {
|
||||
let hash = 2166136261
|
||||
|
||||
for (const character of value) {
|
||||
hash ^= character.codePointAt(0) ?? 0
|
||||
hash = Math.imul(hash, 16777619)
|
||||
}
|
||||
|
||||
return hash >>> 0
|
||||
}
|
||||
|
||||
function countGraphemes(value: string): number {
|
||||
let count = 0
|
||||
|
||||
for (const segment of graphemeSegmenter.segment(value)) {
|
||||
if (segment.segment !== '\n' && segment.segment !== '\r') {
|
||||
count += 1
|
||||
}
|
||||
}
|
||||
|
||||
return count
|
||||
}
|
||||
|
||||
function isKeywordCandidate(segment: string, isWordLike: boolean): boolean {
|
||||
if (!isWordLike || STOP_WORDS.has(segment) || !WORD_PATTERN.test(segment)) {
|
||||
return false
|
||||
}
|
||||
|
||||
const length = countGraphemes(segment)
|
||||
|
||||
if (HAN_PATTERN.test(segment)) {
|
||||
return length >= 1 && length <= 8
|
||||
}
|
||||
|
||||
return length >= 2 && length <= 16
|
||||
}
|
||||
|
||||
function scoreCandidate(
|
||||
text: string,
|
||||
start: number,
|
||||
frequency: number,
|
||||
selectionSeed: number,
|
||||
): number {
|
||||
const length = countGraphemes(text)
|
||||
const lengthScore =
|
||||
length === 2 ? 2.8 : length === 3 ? 2.4 : length === 4 ? 1.8 : 0.7
|
||||
const frequencyScore = Math.min(frequency - 1, 3) * 1.2
|
||||
const hanScore = HAN_PATTERN.test(text) ? 0.8 : 0.3
|
||||
const random = createRandom(hashText(`${text}:${start}:${selectionSeed}`))()
|
||||
|
||||
return lengthScore + frequencyScore + hanScore + random * 1.4
|
||||
}
|
||||
|
||||
export function selectKeywords(
|
||||
text: string,
|
||||
retention: number,
|
||||
selectionSeed = 1,
|
||||
): KeywordCandidate[] {
|
||||
const segments = Array.from(wordSegmenter.segment(text))
|
||||
const candidateSegments = segments.filter((segment) =>
|
||||
isKeywordCandidate(segment.segment, Boolean(segment.isWordLike)),
|
||||
)
|
||||
const frequencies = new Map<string, number>()
|
||||
|
||||
for (const segment of candidateSegments) {
|
||||
frequencies.set(segment.segment, (frequencies.get(segment.segment) ?? 0) + 1)
|
||||
}
|
||||
|
||||
const candidates = candidateSegments.map((segment) => ({
|
||||
id: `${segment.index}:${segment.index + segment.segment.length}`,
|
||||
start: segment.index,
|
||||
end: segment.index + segment.segment.length,
|
||||
text: segment.segment,
|
||||
score: scoreCandidate(
|
||||
segment.segment,
|
||||
segment.index,
|
||||
frequencies.get(segment.segment) ?? 1,
|
||||
selectionSeed,
|
||||
),
|
||||
selected: false,
|
||||
}))
|
||||
|
||||
if (candidates.length === 0) {
|
||||
return candidates
|
||||
}
|
||||
|
||||
const desiredCount =
|
||||
retention <= 0
|
||||
? 0
|
||||
: Math.min(
|
||||
candidates.length,
|
||||
Math.max(1, Math.ceil(candidates.length * (retention / 100))),
|
||||
)
|
||||
const ranked = [...candidates].sort(
|
||||
(left, right) => right.score - left.score || left.start - right.start,
|
||||
)
|
||||
const selectedIds = new Set(
|
||||
ranked.slice(0, desiredCount).map((candidate) => candidate.id),
|
||||
)
|
||||
|
||||
return candidates.map((candidate) => ({
|
||||
...candidate,
|
||||
selected: selectedIds.has(candidate.id),
|
||||
}))
|
||||
}
|
||||
|
||||
function choose<T>(values: readonly T[], random: Random): T {
|
||||
return values[Math.floor(random() * values.length)]
|
||||
}
|
||||
|
||||
function chooseNoiseGlyph(random: Random): string {
|
||||
const categoryRoll = random() * 100
|
||||
|
||||
if (categoryRoll < 60) {
|
||||
return choose(SIMPLE_HAN, random)
|
||||
}
|
||||
|
||||
if (categoryRoll < 80) {
|
||||
return choose(GLITCH_SYMBOLS, random)
|
||||
}
|
||||
|
||||
if (categoryRoll < 90) {
|
||||
return choose(FOREIGN_GLYPHS, random)
|
||||
}
|
||||
|
||||
if (categoryRoll < 97) {
|
||||
return choose(ENCODING_FRAGMENTS, random)
|
||||
}
|
||||
|
||||
return choose(COMPLEX_HAN, random)
|
||||
}
|
||||
|
||||
function generateNoise(length: number, disorder: number, random: Random): string {
|
||||
let output = ''
|
||||
let previous = ''
|
||||
const repeatChance = 0.02 + disorder * 0.0015
|
||||
|
||||
for (let index = 0; index < length; index += 1) {
|
||||
const shouldRepeat = previous !== '' && random() < repeatChance
|
||||
const glyph = shouldRepeat ? previous : chooseNoiseGlyph(random)
|
||||
|
||||
output += glyph
|
||||
previous = glyph
|
||||
}
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
function pushHiddenParts(parts: OutputPart[], text: string): void {
|
||||
const lines = text.split(/(\r?\n)/)
|
||||
|
||||
for (const line of lines) {
|
||||
if (line === '\n' || line === '\r\n') {
|
||||
parts.push({
|
||||
type: 'newline',
|
||||
text: '\n',
|
||||
originalLength: 0,
|
||||
noiseLength: 0,
|
||||
weight: 0,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
const length = countGraphemes(line)
|
||||
|
||||
if (length > 0) {
|
||||
parts.push({
|
||||
type: 'hidden',
|
||||
text: line,
|
||||
originalLength: length,
|
||||
noiseLength: 0,
|
||||
weight: 0,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function buildOutputParts(
|
||||
text: string,
|
||||
retainedRanges: KeywordCandidate[],
|
||||
): OutputPart[] {
|
||||
const parts: OutputPart[] = []
|
||||
let cursor = 0
|
||||
|
||||
for (const range of retainedRanges) {
|
||||
if (range.start > cursor) {
|
||||
pushHiddenParts(parts, text.slice(cursor, range.start))
|
||||
}
|
||||
|
||||
parts.push({
|
||||
type: 'retained',
|
||||
text: text.slice(range.start, range.end),
|
||||
originalLength: countGraphemes(text.slice(range.start, range.end)),
|
||||
noiseLength: 0,
|
||||
weight: 0,
|
||||
})
|
||||
cursor = range.end
|
||||
}
|
||||
|
||||
if (cursor < text.length) {
|
||||
pushHiddenParts(parts, text.slice(cursor))
|
||||
}
|
||||
|
||||
return parts
|
||||
}
|
||||
|
||||
function distributeNoiseBudget(
|
||||
parts: OutputPart[],
|
||||
budget: number,
|
||||
disorder: number,
|
||||
random: Random,
|
||||
): void {
|
||||
const hiddenParts = parts.filter(
|
||||
(part) => part.type === 'hidden' && part.originalLength > 0,
|
||||
)
|
||||
|
||||
if (hiddenParts.length === 0 || budget <= 0) {
|
||||
return
|
||||
}
|
||||
|
||||
const baseAllocation = budget >= hiddenParts.length ? 1 : 0
|
||||
let remaining = budget - baseAllocation * hiddenParts.length
|
||||
const variability = disorder / 100
|
||||
|
||||
for (const part of hiddenParts) {
|
||||
const localScale = 1 + (random() * 2 - 1) * variability * 0.62
|
||||
part.weight = Math.max(0.05, part.originalLength * localScale)
|
||||
part.noiseLength = baseAllocation
|
||||
}
|
||||
|
||||
if (remaining <= 0) {
|
||||
for (let index = 0; index < budget; index += 1) {
|
||||
hiddenParts[index].noiseLength = 1
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
const totalWeight = hiddenParts.reduce((sum, part) => sum + part.weight, 0)
|
||||
const allocations = hiddenParts.map((part) => {
|
||||
const exact = (part.weight / totalWeight) * remaining
|
||||
const whole = Math.floor(exact)
|
||||
part.noiseLength += whole
|
||||
return { part, fraction: exact - whole }
|
||||
})
|
||||
const allocated = allocations.reduce(
|
||||
(sum, allocation) => sum + Math.floor((allocation.part.weight / totalWeight) * remaining),
|
||||
0,
|
||||
)
|
||||
|
||||
remaining -= allocated
|
||||
allocations.sort((left, right) => right.fraction - left.fraction)
|
||||
|
||||
for (let index = 0; index < remaining; index += 1) {
|
||||
allocations[index % allocations.length].part.noiseLength += 1
|
||||
}
|
||||
}
|
||||
|
||||
export function generateMosaic(
|
||||
text: string,
|
||||
candidates: KeywordCandidate[],
|
||||
options: MosaicOptions,
|
||||
): MosaicResult {
|
||||
const random = createRandom(options.seed ^ hashText(text))
|
||||
const retainedRanges = candidates
|
||||
.filter((candidate) => candidate.selected)
|
||||
.sort((left, right) => left.start - right.start)
|
||||
const parts = buildOutputParts(text, retainedRanges)
|
||||
const inputLength = countGraphemes(text)
|
||||
const retainedLength = parts
|
||||
.filter((part) => part.type === 'retained')
|
||||
.reduce((sum, part) => sum + part.originalLength, 0)
|
||||
const jitter = (random() * 2 - 1) * (options.disorder / 100) * 0.035
|
||||
const targetLength = Math.max(
|
||||
retainedLength,
|
||||
Math.round(inputLength * (options.volume / 100 + jitter)),
|
||||
)
|
||||
const noiseBudget = Math.max(0, targetLength - retainedLength)
|
||||
|
||||
distributeNoiseBudget(parts, noiseBudget, options.disorder, random)
|
||||
|
||||
const output = parts
|
||||
.map((part) => {
|
||||
if (part.type === 'hidden') {
|
||||
return generateNoise(part.noiseLength, options.disorder, random)
|
||||
}
|
||||
|
||||
return part.text
|
||||
})
|
||||
.join('')
|
||||
|
||||
return {
|
||||
output,
|
||||
inputLength,
|
||||
outputLength: countGraphemes(output),
|
||||
retainedLength,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user