safe.gno
8.74 Kb · 329 lines
1// Package safe screens short public texts, such as radio dedications, with
2// no human review. A text must use plain characters, carry no link or phone
3// number, and contain no blocked word or phrase, after folding accents, leet
4// digits (0→o, 1→i, 3→e, 4→a, 5→s, 7→t) and repeated letters. Words match
5// whole, without a final s, reversed, or glued back when split by - or '
6// ("fu-ck"); a run of single letters ("f u c k") reads as one word; a few
7// unambiguous stems match inside any word. So "classic" or "grass" pass; the
8// realm adds reports and automatic hiding for what a list cannot know.
9package safe
10
11import "strings"
12
13type list struct {
14 words map[string]bool // false: not blocked, but starts a blocked phrase
15 stems []string // blocked inside any word: roots legitimate words lack
16 back []string // the stems reversed, for words written backwards ("kcuf")
17 first [32]bool // letters (r&31) that start a stem or a reversed stem
18}
19
20// newList reverses the stems once. A reversed "nigg" is too often a real word
21// (jogging): the list has "reggin".
22func newList(words map[string]bool, stems ...string) list {
23 l := list{words: words, stems: stems}
24 for _, st := range stems {
25 l.first[st[0]&31] = true
26 if st != "nigg" {
27 b := reverse(st)
28 l.back = append(l.back, b)
29 l.first[b[0]&31] = true
30 }
31 }
32 return l
33}
34
35// maxPhrase is the longest blocked phrase, in words ("go back where you came from").
36const maxPhrase = 6
37
38var (
39 full = newList(set(words, "\n"), "fuck", "cunt", "nigg", "fagg", "bitch", "whore", "porn", "encul", "pedoph", "paedoph", "pedofil")
40 // hate is what Slur refuses even in titles and names.
41 hate = newList(set("kike,chink,gook,wetback,raghead,towelhead,tranny,trannies,zipperhead,jungle bunny,porch monkey,untermensch,holohoax,heil hitler,sieg heil,white power,gas the jews,kill the jews,kill all jews", ","), "nigg", "fagg")
42 // allow holds real words that a stem, the final s or repeated-letter
43 // folding would catch.
44 allow = set("rapping,lull,sleet,annal,annals,annales,fodder,cull,scunthorpe,bitche,pornic,pornichet,salopette,salopettes", ",")
45)
46
47func set(s, sep string) map[string]bool {
48 m := map[string]bool{}
49 for _, w := range strings.Split(s, sep) {
50 if w == "" {
51 continue
52 }
53 m[w] = true
54 if i := strings.IndexByte(w, ' '); i > 0 && !m[w[:i]] {
55 m[w[:i]] = false
56 }
57 }
58 return m
59}
60
61// Note returns why s cannot be shown publicly ("" when it can). max is the
62// longest text accepted, in characters.
63func Note(s string, max int) string {
64 if len(s) > 4*max {
65 return "too long"
66 }
67 if strings.TrimSpace(s) != s || strings.Contains(s, " ") {
68 return "no leading, trailing or double spaces"
69 }
70 // seq counts digits up to the next letter ("06 12 34 56 78"); dot is 1
71 // after a dot, plus the letters since.
72 n, digits, seq, dot := 0, 0, 0, 0
73 for _, r := range s {
74 n++
75 switch {
76 case r >= '0' && r <= '9':
77 digits++
78 seq++
79 if digits >= 6 || seq >= 9 {
80 return "no phone numbers"
81 }
82 dot = 0
83 case letter(r):
84 if dot > 0 {
85 if dot++; dot > 2 { // x.com, bit.ly, but not P.S.
86 return "no links"
87 }
88 }
89 digits, seq = 0, 0
90 case strings.ContainsRune(" .,!?'-’¡¿", r):
91 digits, dot = 0, 0
92 if r == '.' {
93 dot = 1
94 }
95 default:
96 return "letters, digits and . , ! ? ¡ ¿ ' - only"
97 }
98 }
99 if n > max {
100 return "too long"
101 }
102 if Blocked(s) {
103 return "please rephrase"
104 }
105 return ""
106}
107
108// Blocked reports whether s holds a blocked word, phrase or stem.
109func Blocked(s string) bool { return full.in(s) }
110
111// Slur reports whether s holds a racial or homophobic slur or a hate
112// slogan: the check for titles, names and bios, where the full list would
113// refuse real works (Sex Pistols, Bitches Brew).
114func Slur(s string) bool { return hate.in(s) }
115
116// in checks s folded, then, if it holds a word split by - or ' ("fu-ck") or
117// 1 ! ß inside a word, folded the other way ("b!tch", "ßitch").
118func (l list) in(s string) bool {
119 f, odd := fold(s, false)
120 if l.blocked(f) {
121 return true
122 }
123 if !odd {
124 return false
125 }
126 f, _ = fold(s, true)
127 return l.blocked(f)
128}
129
130func (l list) blocked(s string) bool {
131 ws := fields(s)
132 run := ""
133 for i, w := range append(ws, "") {
134 if _, starts := l.words[w]; starts {
135 for j := i + 2; j <= i+maxPhrase && j <= len(ws); j++ {
136 if l.words[strings.Join(ws[i:j], " ")] {
137 return true
138 }
139 }
140 }
141 if len(w) == 1 {
142 run += w
143 continue
144 }
145 if l.word(run) || l.word(w) {
146 return true
147 }
148 run = ""
149 }
150 return false
151}
152
153// word reports whether w or its letters squeezed is listed, or whether w
154// holds a stem, also reversed ("kcuf"). A reversed listed word is too often
155// a real one (lana, setup).
156func (l list) word(w string) bool {
157 if w == "" || allow[w] {
158 return false
159 }
160 if l.listed(w) {
161 return true
162 }
163 // One pass finds repeated letters (else w is its own squeeze) and a
164 // letter that starts a stem (else no stem can match).
165 dup, stem := false, false
166 var last rune
167 for _, r := range w {
168 dup = dup || r == last
169 stem = stem || l.first[r&31]
170 last = r
171 }
172 sq := w
173 if dup {
174 sq = squeeze(w)
175 if l.listed(sq) {
176 return true
177 }
178 }
179 if len(w) < 4 || !stem {
180 return false
181 }
182 for _, st := range l.stems {
183 if has(w, st) || dup && has(sq, st) {
184 return true
185 }
186 }
187 for _, st := range l.back {
188 if has(w, st) {
189 return true
190 }
191 }
192 return false
193}
194
195// listed matches w whole or without a final s (never "es": spices, not spic).
196func (l list) listed(w string) bool {
197 n := len(w)
198 return l.words[w] || n > 1 && w[n-1] == 's' && l.words[w[:n-1]]
199}
200
201// The helpers below run on every character of every checked text, so they
202// stick to what the VM does cheaply: ranging over a string and writing into
203// a byte slice (indexing a string or appending costs several times more).
204
205// has is strings.Contains for short words (Contains hashes both strings).
206func has(w, st string) bool {
207 c, n := rune(st[0]), len(st)
208 for i, r := range w {
209 if r == c && i+n <= len(w) && w[i:i+n] == st {
210 return true
211 }
212 }
213 return false
214}
215
216// fields is strings.Fields for folded text (lowercase letters and spaces).
217func fields(s string) []string {
218 var ws []string
219 start := -1
220 for i, r := range s {
221 if r != ' ' {
222 if start < 0 {
223 start = i
224 }
225 } else if start >= 0 {
226 ws = append(ws, s[start:i])
227 start = -1
228 }
229 }
230 if start >= 0 {
231 ws = append(ws, s[start:])
232 }
233 return ws
234}
235
236func reverse(w string) string {
237 b := []byte(w)
238 for i, j := 0, len(b)-1; i < j; i, j = i+1, j-1 {
239 b[i], b[j] = b[j], b[i]
240 }
241 return string(b)
242}
243
244func letter(r rune) bool {
245 return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= 0xC0 && r <= 0x17F && r != 0xD7 && r != 0xF7)
246}
247
248// latin folds U+00C0..U+00FF to ASCII ("-" for × and ÷, which letter rejects),
249// latinA U+0100..U+017F ("-" for IJ ij Œ œ, folded to two letters).
250const (
251 latin = "aaaaaaaceeeeiiiidnooooo-ouuuuyts" + "aaaaaaaceeeeiiiidnooooo-ouuuuyty"
252 latinA = "aaaaaaccccccccddddeeeeeeeeeegggg" + "gggghhhhiiiiiiiiii--jjkkklllllll" +
253 "lllnnnnnnnnnoooooo--rrrrrrssssss" + "ssttttttuuuuuuuuuuuuwwyyyzzzzzzs"
254)
255
256// cyrillic letters that look Latin, and what they read as (2 bytes each).
257const cyrillic, cyrillicAs = "аеорсухіјѕһԁԛԝӏАВЕНІЈКМОРСТХЅ", "aeopcyxijshdqwlabehijkmopctxs"
258
259// fold lowercases s to ASCII letters and spaces: accents dropped, look-alike
260// Cyrillic and leet digits read as letters, anything else a space. alt glues
261// words split by - or ' and reads 1 ! ß as l i b (1 and ! before a letter);
262// odd reports whether alt reads s differently.
263func fold(s string, alt bool) (f string, odd bool) {
264 b := make([]byte, len(s)) // folding never lengthens s
265 n := 0
266 for i, r := range s {
267 if r >= 'a' && r <= 'z' || r == ' ' {
268 b[n] = byte(r)
269 n++
270 continue
271 }
272 if r >= 'A' && r <= 'Z' {
273 b[n] = byte(r) + 32
274 n++
275 continue
276 }
277 if r == '-' || r == '\'' || r == '’' || r == 'ß' || (r == '1' || r == '!') && i+1 < len(s) && letter(rune(s[i+1])) {
278 odd = true
279 if alt {
280 if j := strings.IndexRune("1!ß", r); j >= 0 {
281 b[n] = "lib"[j]
282 n++
283 }
284 continue
285 }
286 }
287 switch {
288 case r >= 0xC0 && r <= 0xFF:
289 if c := latin[r-0xC0]; c != '-' {
290 b[n] = c
291 n++
292 }
293 case r == 'IJ' || r == 'ij':
294 b[n], b[n+1] = 'i', 'j'
295 n += 2
296 case r == 'Œ' || r == 'œ':
297 b[n], b[n+1] = 'o', 'e'
298 n += 2
299 case r >= 0x100 && r <= 0x17F:
300 b[n] = latinA[r-0x100]
301 n++
302 case r >= '0' && r <= '9':
303 b[n] = "oizeasgtbg"[r-'0']
304 n++
305 case r >= 0x400 && r <= 0x52F && strings.ContainsRune(cyrillic, r):
306 b[n] = cyrillicAs[strings.IndexRune(cyrillic, r)/2]
307 n++
308 default:
309 b[n] = ' '
310 n++
311 }
312 }
313 return string(b[:n]), odd
314}
315
316// squeeze collapses repeated letters ("fuuuck" -> "fuck").
317func squeeze(w string) string {
318 b := []byte(w)
319 n := 0
320 var last byte
321 for i, c := range b {
322 if i == 0 || c != last {
323 b[n] = c
324 n++
325 }
326 last = c
327 }
328 return string(b[:n])
329}
