ref_token.bend source
ref_token.bend on the hub · documented module
# REFERENCE_TOKEN-shaped scanners (grab-bcv share.ts / bible-linkify spirit).# Pure Bend — helper-pair style, Nat-fuel where looping, no @unsafe.import Basedef char1(+c: Char) -> String: String.from_list(c <> Nil{})def is_alnum(+c: Char) -> Bool: Bool.or(Char.is_alpha(c), Char.is_digit(c))def is_book_num(+c: Char) -> Bool: Bool.or( Char.is_eq(c, '1'), Bool.or(Char.is_eq(c, '2'), Char.is_eq(c, '3')) )def drop_head(+s: String) -> String: match s: case SNil{}: "" case SCon{h, t}: tdef longer_tok_go(ge: Bool, +a: String, +b: String) -> String: match ge: case True{}: a case False{}: bdef longer_tok(+a: String, +b: String) -> String: longer_tok_go(Nat.is_ge(String.length(a), String.length(b)), a, b)def pick_nonempty_b(b_empty: Bool, +a: String, +b: String) -> String: match b_empty: case True{}: a case False{}: longer_tok(a, b)def pick_nonempty_a(a_empty: Bool, +a: String, +b: String) -> String: match a_empty: case True{}: b case False{}: pick_nonempty_b(String.is_empty(b), a, b)def pick_nonempty(+a: String, +b: String) -> String: pick_nonempty_a(String.is_empty(a), a, b)# --- take alphas / digits / alnum ---type TkPh is Data: TkCheck{} TkDec{ok: Bool} TkTake{}def take_alphas_go(fuel: Nat, ph: TkPh, rest: String, acc: String) -> String & String: match fuel: case 0n: (acc, rest) case 1n+p: match ph: case TkCheck{}: match rest: case SNil{}: (acc, "") case SCon{+h, +t}: take_alphas_go(p, TkDec{Char.is_alpha(h)}, rest, acc) case TkDec{ok}: match ok: case False{}: (acc, rest) case True{}: take_alphas_go(p, TkTake{}, rest, acc) case TkTake{}: match rest: case SNil{}: (acc, "") case SCon{+h, +t}: take_alphas_go(p, TkCheck{}, t, String.append(acc, char1(h)))def take_alphas(+s: String) -> String & String: take_alphas_go(Nat.add(Nat.mul(3n, String.length(s)), 3n), TkCheck{}, s, "")def take_digits_go(fuel: Nat, ph: TkPh, rest: String, acc: String) -> String & String: match fuel: case 0n: (acc, rest) case 1n+p: match ph: case TkCheck{}: match rest: case SNil{}: (acc, "") case SCon{+h, +t}: take_digits_go(p, TkDec{Char.is_digit(h)}, rest, acc) case TkDec{ok}: match ok: case False{}: (acc, rest) case True{}: take_digits_go(p, TkTake{}, rest, acc) case TkTake{}: match rest: case SNil{}: (acc, "") case SCon{+h, +t}: take_digits_go(p, TkCheck{}, t, String.append(acc, char1(h)))def take_digits(+s: String) -> String & String: take_digits_go(Nat.add(Nat.mul(3n, String.length(s)), 3n), TkCheck{}, s, "")def take_alnum_go(fuel: Nat, ph: TkPh, rest: String, acc: String) -> String & String: match fuel: case 0n: (acc, rest) case 1n+p: match ph: case TkCheck{}: match rest: case SNil{}: (acc, "") case SCon{+h, +t}: take_alnum_go(p, TkDec{is_alnum(h)}, rest, acc) case TkDec{ok}: match ok: case False{}: (acc, rest) case True{}: take_alnum_go(p, TkTake{}, rest, acc) case TkTake{}: match rest: case SNil{}: (acc, "") case SCon{+h, +t}: take_alnum_go(p, TkCheck{}, t, String.append(acc, char1(h)))def take_alnum(+s: String) -> String & String: take_alnum_go(Nat.add(Nat.mul(3n, String.length(s)), 3n), TkCheck{}, s, "")def space_pair(sp: Bool, +t: String, +orig: String) -> Bool & String: match sp: case True{}: (True{}, t) case False{}: (False{}, orig)def drop_one_space(+s: String) -> Bool & String: match s: case SNil{}: (False{}, "") case SCon{+h, +t}: space_pair(Char.is_space(h), t, s)# Zero-or-more spaces (grab-bcv \s* after numbered-book digit).def take_spaces_go(fuel: Nat, ph: TkPh, rest: String, acc: String) -> String & String: match fuel: case 0n: (acc, rest) case 1n+p: match ph: case TkCheck{}: match rest: case SNil{}: (acc, "") case SCon{+h, +t}: take_spaces_go(p, TkDec{Char.is_space(h)}, rest, acc) case TkDec{ok}: match ok: case False{}: (acc, rest) case True{}: take_spaces_go(p, TkTake{}, rest, acc) case TkTake{}: match rest: case SNil{}: (acc, "") case SCon{+h, +t}: take_spaces_go(p, TkCheck{}, t, String.append(acc, char1(h)))def take_spaces(+s: String) -> String & String: take_spaces_go(Nat.add(Nat.mul(3n, String.length(s)), 3n), TkCheck{}, s, "")def char_eq_head(+s: String, +c: Char) -> Bool: match s: case SNil{}: False{} case SCon{+h, t}: Char.is_eq(h, c)def dig_head(+s: String) -> Bool: match s: case SNil{}: False{} case SCon{+h, t}: Char.is_digit(h)def alpha_head(+s: String) -> Bool: match s: case SNil{}: False{} case SCon{+h, t}: Char.is_alpha(h)# ===== Human token =====# ([1-3]\s*)?[A-Za-z]+\.?(?:\s+of\s+[A-Za-z]+)?\s+\d+(?:(?::|\s)\d+(?:-\d+)?)?# Numbered book: grab-bcv REFERENCE_TOKEN uses \s* (0+ spaces) — allow "1John 3:16" and "1 John 3:16".def hu_fail() -> String: ""def hu_finish(+tok: String) -> String: tokdef hu_range_empty(empty: Bool, +pre: String, +ed: String) -> String: match empty: case True{}: hu_finish(pre) case False{}: hu_finish(String.append(pre, ed))def hu_range_pair(+pre: String, pair: String & String) -> String: match pair: case (+ed, rest2): hu_range_empty(String.is_empty(ed), pre, ed)def hu_range(+pre: String, +rest: String) -> String: hu_range_pair(pre, take_digits(rest))def hu_after_verse_dash(dash: Bool, +pre: String, +rest: String) -> String: match dash: case True{}: hu_range(String.append(pre, "-"), drop_head(rest)) case False{}: hu_finish(pre)def hu_after_verse(+pre: String, +rest: String) -> String: hu_after_verse_dash(char_eq_head(rest, '-'), pre, rest)def hu_verse_empty(empty: Bool, +pre: String, +vs: String, +rest: String) -> String: match empty: case True{}: hu_finish(pre) case False{}: hu_after_verse(String.append(pre, vs), rest)def hu_verse_pair(+pre: String, pair: String & String) -> String: match pair: case (+vs, +rest2): hu_verse_empty(String.is_empty(vs), pre, vs, rest2)def hu_verse(+pre: String, +rest: String) -> String: hu_verse_pair(pre, take_digits(rest))def hu_cv_space_dig(dig: Bool, +pre: String, +rest: String) -> String: match dig: case True{}: hu_verse(String.append(pre, " "), drop_head(rest)) case False{}: hu_finish(pre)def hu_cv_space(+pre: String, +rest: String) -> String: hu_cv_space_dig(dig_head(drop_head(rest)), pre, rest)def hu_cv_colon(colon: Bool, space: Bool, glued: Bool, +pre: String, +rest: String) -> String: match colon: case True{}: hu_verse(String.append(pre, ":"), drop_head(rest)) case False{}: match space: case True{}: hu_cv_space(pre, rest) case False{}: match glued: case True{}: hu_fail() case False{}: hu_finish(pre)def hu_cv_heads(+pre: String, +rest: String, +h: Char) -> String: hu_cv_colon(Char.is_eq(h, ':'), Char.is_space(h), is_alnum(h), pre, rest)def hu_after_chap(+pre: String, +rest: String) -> String: match rest: case SNil{}: hu_finish(pre) case SCon{+h, t}: hu_cv_heads(pre, rest, h)def hu_chap_empty(empty: Bool, +pre: String, +ch: String, +rest: String) -> String: match empty: case True{}: hu_fail() case False{}: hu_after_chap(String.append(pre, ch), rest)def hu_chap_pair(+pre: String, pair: String & String) -> String: match pair: case (+ch, +rest2): hu_chap_empty(String.is_empty(ch), pre, ch, rest2)def hu_chap(+pre: String, +rest: String) -> String: hu_chap_pair(pre, take_digits(rest))def hu_need_chap_sp_go(sp: Bool, +pre: String, +rest: String) -> String: match sp: case False{}: hu_fail() case True{}: hu_chap(String.append(pre, " "), rest)def hu_need_chap_sp(+pre: String, pair: Bool & String) -> String: match pair: case (sp, rest2): hu_need_chap_sp_go(sp, pre, rest2)def hu_of_book_empty(empty: Bool, +pre: String, +b2: String, +rest: String) -> String: match empty: case True{}: hu_fail() case False{}: hu_need_chap_sp(String.append(pre, b2), drop_one_space(rest))def hu_of_book_pair(+pre: String, pair: String & String) -> String: match pair: case (+b2, +rest2): hu_of_book_empty(String.is_empty(b2), pre, b2, rest2)def hu_of_book(+pre: String, +rest: String) -> String: hu_of_book_pair(pre, take_alphas(rest))def hu_of_sp2_go(sp: Bool, +pre: String, +rest: String) -> String: match sp: case False{}: hu_fail() case True{}: hu_of_book(String.append(pre, " "), rest)def hu_of_sp2(+pre: String, pair: Bool & String) -> String: match pair: case (sp, rest2): hu_of_sp2_go(sp, pre, rest2)def hu_after_of(+pre: String, +rest: String) -> String: hu_of_sp2(pre, drop_one_space(rest))def hu_of_word_go(is_of: Bool, +pre: String, +rest: String) -> String: match is_of: case True{}: hu_after_of(String.append(pre, " of"), drop_head(drop_head(rest))) case False{}: hu_chap(String.append(pre, " "), rest)def hu_of_word(+pre: String, +rest: String) -> String: hu_of_word_go(String.starts_with(String.to_lower(rest), "of"), pre, rest)def hu_of_sp_go(sp: Bool, +pre: String, +rest: String) -> String: match sp: case False{}: hu_fail() case True{}: hu_of_word(pre, rest)def hu_of_sp(+pre: String, pair: Bool & String) -> String: match pair: case (sp, rest2): hu_of_sp_go(sp, pre, rest2)def hu_after_dot(+pre: String, +rest: String) -> String: hu_of_sp(pre, drop_one_space(rest))def hu_dot_go(dotted: Bool, +pre: String, +rest: String) -> String: match dotted: case True{}: hu_after_dot(String.append(pre, "."), drop_head(rest)) case False{}: hu_after_dot(pre, rest)def hu_after_book(+pre: String, +rest: String) -> String: hu_dot_go(char_eq_head(rest, '.'), pre, rest)def hu_book_empty(empty: Bool, +pre: String, +book: String, +rest: String) -> String: match empty: case True{}: hu_fail() case False{}: hu_after_book(String.append(pre, book), rest)def hu_book_pair(+pre: String, pair: String & String) -> String: match pair: case (+book, +rest2): hu_book_empty(String.is_empty(book), pre, book, rest2)def hu_book(+pre: String, +rest: String) -> String: hu_book_pair(pre, take_alphas(rest))def hu_num_alpha_go(ok: Bool, +pre: String, +rest: String) -> String: match ok: case False{}: hu_fail() case True{}: hu_book(pre, rest)def hu_num_spaces_pair(+pre: String, pair: String & String) -> String: match pair: case (+sp, +rest2): hu_num_alpha_go(alpha_head(rest2), String.append(pre, sp), rest2)def hu_num_spaces(+pre: String, +rest: String) -> String: hu_num_spaces_pair(pre, take_spaces(rest))def hu_start_num(num: Bool, +h: Char, +t: String, +rest: String) -> String: match num: case True{}: hu_num_spaces(char1(h), t) case False{}: hu_book("", rest)def match_human(+s: String) -> String: match s: case SNil{}: "" case SCon{+h, +t}: hu_start_num(is_book_num(h), h, t, s)# ===== OSIS token =====# [1-3]?[A-Za-z]{2,}\.\d+(?:\.\d+)?(?:-\d+)?(?:\.[A-Za-z0-9]{2,8})?def os_fail() -> String: ""def os_finish(+tok: String) -> String: tokdef suf_ok(+suf: String) -> Bool: Bool.and( Nat.is_ge(String.length(suf), 2n), Nat.is_le(String.length(suf), 8n) )def os_suf_body_go(ok: Bool, +pre: String, +suf: String) -> String: match ok: case True{}: os_finish(String.append(pre, suf)) case False{}: os_finish(pre)def os_suf_body_pair(+pre: String, pair: String & String) -> String: match pair: case (+suf, rest2): os_suf_body_go(suf_ok(suf), pre, suf)def os_suf_body(+pre: String, +rest: String) -> String: os_suf_body_pair(pre, take_alnum(rest))def os_suf_go(dotted: Bool, +pre: String, +rest: String) -> String: match dotted: case True{}: os_suf_body(String.append(pre, "."), drop_head(rest)) case False{}: os_finish(pre)def os_suf(+pre: String, +rest: String) -> String: os_suf_go(char_eq_head(rest, '.'), pre, rest)# End-half of an OSIS range (no nested dash) — avoids mutual recursion with os_book.def os_end_verse_empty(empty: Bool, +pre: String, +vs: String, +rest: String) -> String: match empty: case True{}: os_fail() case False{}: os_suf(String.append(pre, vs), rest)def os_end_verse_pair(+pre: String, pair: String & String) -> String: match pair: case (+vs, +rest2): os_end_verse_empty(String.is_empty(vs), pre, vs, rest2)def os_end_verse(+pre: String, +rest: String) -> String: os_end_verse_pair(pre, take_digits(rest))def os_end_vdot_go(dotted: Bool, +pre: String, +rest: String) -> String: match dotted: case True{}: os_end_verse(String.append(pre, "."), drop_head(rest)) case False{}: os_suf(pre, rest)def os_end_after_chap(+pre: String, +rest: String) -> String: os_end_vdot_go(char_eq_head(rest, '.'), pre, rest)def os_end_chap_empty(empty: Bool, +pre: String, +ch: String, +rest: String) -> String: match empty: case True{}: os_fail() case False{}: os_end_after_chap(String.append(pre, ch), rest)def os_end_chap_pair(+pre: String, pair: String & String) -> String: match pair: case (+ch, +rest2): os_end_chap_empty(String.is_empty(ch), pre, ch, rest2)def os_end_chap(+pre: String, +rest: String) -> String: os_end_chap_pair(pre, take_digits(rest))def os_end_dot_go(dotted: Bool, +pre: String, +rest: String) -> String: match dotted: case False{}: os_fail() case True{}: os_end_chap(String.append(pre, "."), drop_head(rest))def os_end_after_book(+pre: String, +rest: String) -> String: os_end_dot_go(char_eq_head(rest, '.'), pre, rest)def os_end_book_short(short: Bool, +pre: String, +book: String, +rest: String) -> String: match short: case True{}: os_fail() case False{}: os_end_after_book(String.append(pre, book), rest)def os_end_book_pair(+pre: String, pair: String & String) -> String: match pair: case (+book, +rest2): os_end_book_short(Nat.is_lt(String.length(book), 2n), pre, book, rest2)def os_end_book(+pre: String, +rest: String) -> String: os_end_book_pair(pre, take_alphas(rest))def os_end_num_alpha(alpha: Bool, +pre: String, +rest: String) -> String: match alpha: case True{}: os_end_book(pre, rest) case False{}: os_fail()def os_end_start_num(num: Bool, +h: Char, +t: String, +pre: String) -> String: match num: case True{}: os_end_num_alpha(alpha_head(t), String.append(pre, char1(h)), t) case False{}: os_end_book(pre, SCon{h, t})def os_end_ref(+pre: String, +rest: String) -> String: match rest: case SNil{}: os_fail() case SCon{+h, +t}: os_end_start_num(is_book_num(h), h, t, pre)def os_range_end_verse_empty(empty: Bool, +pre: String, +vs: String, +rest: String) -> String: match empty: case True{}: os_fail() case False{}: os_suf(String.append(pre, vs), rest)def os_range_end_verse_pair(+pre: String, pair: String & String) -> String: match pair: case (+vs, +rest2): os_range_end_verse_empty(String.is_empty(vs), pre, vs, rest2)def os_range_end_verse(+pre: String, +rest: String) -> String: os_range_end_verse_pair(pre, take_digits(rest))def os_range_after_num_go(dotted: Bool, +pre: String, +rest: String) -> String: match dotted: case True{}: os_range_end_verse(String.append(pre, "."), drop_head(rest)) case False{}: os_suf(pre, rest)def os_range_after_num(+pre: String, +rest: String) -> String: os_range_after_num_go(char_eq_head(rest, '.'), pre, rest)def os_range_dig_go(empty: Bool, +pre: String, +ed: String, +rest2: String) -> String: match empty: case True{}: os_fail() case False{}: os_range_after_num(String.append(pre, ed), rest2)def os_range_dig_pair(+pre: String, pair: String & String) -> String: match pair: case (+ed, +rest2): os_range_dig_go(String.is_empty(ed), pre, ed, rest2)def os_range_dig(+pre: String, +rest: String) -> String: os_range_dig_pair(pre, take_digits(rest))def bookish_head(+s: String) -> Bool: match s: case SNil{}: False{} case SCon{+h, +t}: Bool.or(Char.is_alpha(h), Bool.and(is_book_num(h), alpha_head(t)))def os_range_dig_or_fail(dig: Bool, +pre: String, +rest: String) -> String: match dig: case True{}: os_range_dig(pre, rest) case False{}: os_fail()def os_range_go(bookish: Bool, +pre: String, +rest: String) -> String: match bookish: case True{}: os_end_ref(pre, rest) case False{}: os_range_dig_or_fail(dig_head(rest), pre, rest)def os_try_range_go(empty: Bool, +with_dash: String, +bare: String) -> String: match empty: case True{}: bare case False{}: with_dashdef os_try_range_att(+pre: String, +att: String) -> String: os_try_range_go(String.is_empty(att), att, os_finish(pre))def os_try_range(+pre: String, +rest: String) -> String: os_try_range_att(pre, os_range_go(bookish_head(rest), String.append(pre, "-"), rest))def os_after_dash(+pre: String, +rest: String) -> String: os_try_range(pre, rest)def os_dash_go(dash: Bool, +pre: String, +rest: String) -> String: match dash: case True{}: os_after_dash(pre, drop_head(rest)) case False{}: os_suf(pre, rest)def os_after_verse(+pre: String, +rest: String) -> String: os_dash_go(char_eq_head(rest, '-'), pre, rest)def os_verse_empty(empty: Bool, +pre: String, +vs: String, +rest: String) -> String: match empty: case True{}: os_after_verse(pre, rest) case False{}: os_after_verse(String.append(pre, vs), rest)def os_verse_pair(+pre: String, pair: String & String) -> String: match pair: case (+vs, +rest2): os_verse_empty(String.is_empty(vs), pre, vs, rest2)def os_verse(+pre: String, +rest: String) -> String: os_verse_pair(pre, take_digits(rest))def os_vdot_go(dotted: Bool, +pre: String, +rest: String) -> String: match dotted: case True{}: os_verse(String.append(pre, "."), drop_head(rest)) case False{}: os_after_verse(pre, rest)def os_after_chap(+pre: String, +rest: String) -> String: os_vdot_go(char_eq_head(rest, '.'), pre, rest)def os_chap_empty(empty: Bool, +pre: String, +ch: String, +rest: String) -> String: match empty: case True{}: os_fail() case False{}: os_after_chap(String.append(pre, ch), rest)def os_chap_pair(+pre: String, pair: String & String) -> String: match pair: case (+ch, +rest2): os_chap_empty(String.is_empty(ch), pre, ch, rest2)def os_chap(+pre: String, +rest: String) -> String: os_chap_pair(pre, take_digits(rest))def os_dot_go(dotted: Bool, +pre: String, +rest: String) -> String: match dotted: case False{}: os_fail() case True{}: os_chap(String.append(pre, "."), drop_head(rest))def os_after_book(+pre: String, +rest: String) -> String: os_dot_go(char_eq_head(rest, '.'), pre, rest)def os_book_short(short: Bool, +pre: String, +book: String, +rest: String) -> String: match short: case True{}: os_fail() case False{}: os_after_book(String.append(pre, book), rest)def os_book_pair(+pre: String, pair: String & String) -> String: match pair: case (+book, +rest2): os_book_short(Nat.is_lt(String.length(book), 2n), pre, book, rest2)def os_book(+pre: String, +rest: String) -> String: os_book_pair(pre, take_alphas(rest))def os_num_alpha_go(alpha: Bool, +pre: String, +rest: String, +full: String) -> String: match alpha: case True{}: os_book(pre, rest) case False{}: os_book("", full)def os_start_num(num: Bool, +h: Char, +t: String, +rest: String) -> String: match num: case True{}: os_num_alpha_go(alpha_head(t), char1(h), t, rest) case False{}: os_book("", rest)def match_osis(+s: String) -> String: match s: case SNil{}: "" case SCon{+h, +t}: os_start_num(is_book_num(h), h, t, s)def match_ref_token(+s: String) -> String: pick_nonempty(match_osis(s), match_human(s))# --- URL token ---type UrlPh is Data: UrlCheck{} UrlDec{stop: Bool} UrlTake{}def is_url_stop(+c: Char) -> Bool: Bool.or( Char.is_space(c), Bool.or(Char.is_eq(c, ')'), Char.is_eq(c, ']')) )def take_url_go(fuel: Nat, ph: UrlPh, rest: String, acc: String) -> String: match fuel: case 0n: acc case 1n+p: match ph: case UrlCheck{}: match rest: case SNil{}: acc case SCon{+h, +t}: take_url_go(p, UrlDec{is_url_stop(h)}, rest, acc) case UrlDec{stop}: match stop: case True{}: acc case False{}: take_url_go(p, UrlTake{}, rest, acc) case UrlTake{}: match rest: case SNil{}: acc case SCon{+h, +t}: take_url_go(p, UrlCheck{}, t, String.append(acc, char1(h)))def starts_url(+s: String) -> Bool: Bool.or( String.starts_with(String.to_lower(s), "http://"), Bool.or( String.starts_with(String.to_lower(s), "https://"), String.starts_with(String.to_lower(s), "www.") ) )def match_url_token_go(yes: Bool, +s: String) -> String: match yes: case False{}: "" case True{}: take_url_go( Nat.add(Nat.mul(3n, String.length(s)), 4n), UrlCheck{}, s, "" )def match_url_token(+s: String) -> String: match_url_token_go(starts_url(s), s)def boundary_ok(prev_alnum: Bool) -> Bool: match prev_alnum: case True{}: False{} case False{}: True{}