import re, sys, json, glob, os # Score v2: adds complex_tense (perfect tenses, modal stacks), exempts # adjectival/stative participles from the passive count, moves "provide" to the # banned list, adds a noun-train marker and a --strict mode. The episode's # published numbers were measured with score v1 (this file's git history at the # episode date); v1 and v2 totals are close but not directly comparable. # # Score v3: strips code fences with a line-by-line state machine instead of a # regex (a single unclosed ``` shifted pairing and leaked fenced semicolons # into the prose count), counts possessive 's (user's, thread's) as possessive # not contraction - only pronoun-stem 's (it's, that's, who's) is a # contraction - and exempts double-quoted spans from word-level checks (STE # treats quoted material as verbatim speech). SCORE_VERSION = 3 MARKETING = ["seamless","seamlessly","robust","powerful","cutting-edge","effortless","effortlessly", "world-class","next-generation","revolutionary","blazing","lightning-fast","elegant","delightful", "turnkey","best-in-class","state-of-the-art","game-changing","first-class","battle-tested", "enterprise-grade","supercharge","unlock","unleash","empower","empowers"] BANNED = ["begin","begins","commence","commences","initiate","initiates","originate", "utilize","utilizes","utilizing","leverage","leverages","leveraging","facilitate","facilitates", "ensure","ensures","ensuring","prior to","subsequent to","obtain","obtains","acquire","acquires", "demonstrate","demonstrates","additionally","furthermore","moreover","comprehensive","comprehensively", "utilization","aforementioned","henceforth","therein","whilst","amongst","numerous","myriad","plethora", "provide","provides","provided", "in order to","a variety of","in the event that","due to the fact that","it is important to note"] # STE's own recurring-errors list (see ste-recurring-errors.md). Counted only # with --strict: these are correct STE but would flag normal prose in docs. STRICT_BANNED = ["however","since","should","shall","using","follow","follows","followed"] PHRASAL = ["spin up","spin down","reach out","dive into","dives into","diving into","kick off","kicks off", "roll out","rolls out","tear down","ramp up","circle back","drill down","spun up","reaching out"] MODAL_HEDGE = ["it is important to note","it should be noted","it is worth noting","please note that", "as mentioned","as noted above"] BE = r"(?:am|is|are|was|were|be|been|being)" PP_IRREG = r"(?:done|made|sent|read|built|kept|held|set|put|run|written|shown|given|taken|found|got|gotten|seen|known|thrown|drawn)" # Rule 3.3: a past participle used as an adjective is not passive. These # stative participles only count as passive when a by-agent follows. STATIVE = r"(?:closed|opened?|damaged|completed?|installed|connected|required|expected|configured|enabled|disabled|deprecated|supported)" FUNC_WORDS = set("""a an the this that these those of for to in on at by with from as and or but if when then than not no is are was were be been being am do does did has have had will would can could may might must should shall it its their your our his her they we you i""".split()) def strip_code(t): out = [] in_fence = False for line in t.split("\n"): if re.match(r"^\s*```", line): in_fence = not in_fence continue if not in_fence: out.append(line) t = "\n".join(out) t = re.sub(r"`[^`]*`", " ", t) return t def sentences(text): out = [] for line in text.split("\n"): s = line.strip() if not s: continue s = re.sub(r"^\s*#{1,6}\s*", "", s) s = re.sub(r"^\s*(?:[-*+]|\d+[.)])\s+", "", s) if not s: continue parts = re.split(r"(?<=[.!?:])\s+(?=[A-Z0-9\"'\-])", s) for p in parts: p = p.strip() if p: out.append(p) return out def wc(s): return len([w for w in re.findall(r"[A-Za-z0-9][A-Za-z0-9'\-/]*", s)]) def count_ci(text, phrases): n = 0; hits = [] low = text.lower() for ph in phrases: for m in re.finditer(r"(?= 4: hits.append(" ".join(run)) run = [] return hits def unquote(text): """Exempt double-quoted spans from word-level checks (STE treats quoted material as verbatim). YAML description wrapper quotes are not quoted speech: unwrap them first so descriptions still get checked.""" out = [] for line in text.split("\n"): m = re.match(r'^(\s*[a-z_-]+:\s*)"(.*)"\s*$', line, re.S) if m: line = m.group(1) + m.group(2) res = [] in_q = False for seg in re.split(r'(")', line): if seg == '"': in_q = not in_q continue res.append(" " * len(seg) if in_q else seg) out.append("".join(res)) return "\n".join(out) def lint(text, strict=False): raw = text text = unquote(strip_code(text)) sents = sentences(text) words = sum(wc(s) for s in sents) or 1 v = {} longs = [(wc(s), s) for s in sents if wc(s) > 20] v["long_sentence(>20w)"] = len(longs) v["semicolon"] = text.count(";") v["contraction"] = len(re.findall( r"\b(?:\w+['’](?:t|re|ve|ll|d|m)\b|(?:it|that|who|what|there|here|he|she|let|how|where|when|why)['’]s\b)", text)) passive_parts = re.findall(rf"\b{BE}\s+(\w+ed|{PP_IRREG})\b", text, re.I) v["passive_voice"] = sum(1 for p in passive_parts if not re.fullmatch(STATIVE, p, re.I)) \ + len(re.findall(rf"\b{BE}\s+{STATIVE}\s+by\b", text, re.I)) v["complex_tense"] = len(re.findall( rf"\b(?:(?:may|might|could|would|should|must|will|shall|can)\s+)?(?:have|has|had)\s+(?:been\s+)?(?:\w+ed|{PP_IRREG})\b", text, re.I)) v["ing_main_verb"] = len(re.findall(rf"\b{BE}\s+\w+ing\b", text, re.I)) v["nominalization"] = len(re.findall(r"\b(?:perform(?:s|ed)?|conduct(?:s|ed)?|carry out|carries out|make use of|makes use of)\b", text, re.I)) + len(re.findall(r"\b\w{4,}(?:tion|ment|ance|ence)\s+of\b", text, re.I)) v["phrasal_verb"], _ = count_ci(text, PHRASAL) v["banned_word"], bh = count_ci(text, BANNED) v["marketing_adjective"], mh = count_ci(text, MARKETING) v["modal_hedge"], _ = count_ci(text, MODAL_HEDGE) paras = [p for p in re.split(r"\n\s*\n", text) if p.strip()] def is_list_block(p): lines = [l for l in p.split("\n") if l.strip()] if not lines: return False listy = [l for l in lines if re.match(r"^\s*(?:[-*+]|\d+[.)])\s+", l)] return len(listy) / len(lines) >= 0.5 def is_description(p): first = next((l for l in p.split("\n") if l.strip()), "") return bool(re.match(r"^\s*(?:description|motion_description|additional-description):", first)) v["long_paragraph(>6s)"] = sum(1 for p in paras if not is_description(p) and not is_list_block(p) and len(sentences(strip_code(p))) > 6) em = raw.count("—") + raw.count("–") trains = noun_trains(text) if strict: n_strict, sh = count_ci(text, STRICT_BANNED) # "may" is matched case-sensitively so the month "May" stays clean n_strict += len(re.findall(r"(?=4w,marker)": len(trains), "longest_sentence_words": (max(longs)[0] if longs else max((wc(s) for s in sents), default=0)), "sample_marketing": list(dict.fromkeys(mh))[:6], "sample_banned": list(dict.fromkeys(bh))[:6], "sample_noun_train": trains[:3], } if __name__ == "__main__": args = sys.argv[1:] strict = "--strict" in args as_json = "--json" in args fail_over = None if "--fail-over" in args: i = args.index("--fail-over") fail_over = float(args[i + 1]) del args[i:i + 2] files = [a for a in args if a not in ("--strict", "--json")] worst = 0.0 if not files: sys.stdin.reconfigure(encoding="utf-8") r = lint(sys.stdin.read(), strict=strict) print(json.dumps(r, indent=2)) worst = r["total_per100w"] else: exp = [] for f in files: exp += sorted(glob.glob(f)) if any(c in f for c in "*?[") else [f] for f in exp: with open(f, encoding="utf-8") as fh: r = lint(fh.read(), strict=strict) worst = max(worst, r["total_per100w"]) if as_json: print(json.dumps({"file": f, **r}, indent=2)) else: print(f"{os.path.basename(f):32} words={r['words']:4d} total={r['total']:3d} per100w={r['total_per100w']:6.2f} em_dash={r['em_dash(slop-marker)']:2d}") if fail_over is not None and worst > fail_over: sys.exit(1)