package ltx import "base:runtime" import "core:fmt" import "core:os" import "core:path/filepath" import "core:strings" import "core:unicode" // TODO(Paul): change the key-value format to allow for true arbitrary string values with the use of quotes (maybe?) and make the rules clean and easy to remember. main :: proc() { args := parse_args() if args.type == .None do return ltx: Ltx parse_err := parse_file(<x, args.file_path) defer free_file(<x) if parse_err != .None { fmt.eprintln(get_error(<x, parse_err, context.temp_allocator)) return } sb := strings.builder_make() defer strings.builder_destroy(&sb) switch args.type { case .None: return case .XML: to_xml(&sb, ltx.nodes) case .PlainText: to_plain_text(&sb, ltx.nodes) case .SmartIndentedText: to_text_with_smart_indent(&sb, ltx.nodes) } fmt.println(strings.to_string(sb)) // for node in ltx.nodes { // print_node(node) // } // sb := strings.builder_make() // defer strings.builder_destroy(&sb) // fmt.println("--- BEGIN XML ---") // strings.builder_reset(&sb) // to_xml(&sb, ltx.nodes) // fmt.println(strings.to_string(sb)) // fmt.println("--- END XML ---") // fmt.println("--- BEGIN PLAIN TEXT ---") // strings.builder_reset(&sb) // to_text_with_smart_indent(&sb, ltx.nodes) // fmt.println(strings.to_string(sb)) // fmt.println("--- END PLAIN TEXT ---") } print_help :: proc() { fmt.eprintln( "A naive implementation of TeX-based arbitrary markup. Prints syntax tree and space-preserving XML substitution", ) fmt.eprintln("Usage:") fmt.eprintf("\t%s \n", os.args[0]) } parse_args :: proc() -> Cli_Args { args := Cli_Args{} if len(os.args) < 2 { print_help() return args } output_flags_used := 0 output_type := Cli_Output_Type.None file_path: string for arg in os.args[1:] { if strings.starts_with(arg, "--") { output_flags_used += 1 switch arg[2:] { case "xml": output_type = .XML case "text": output_type = .PlainText case "smart": output_type = .SmartIndentedText case: fmt.eprintln("Unknown flag:", arg) print_help() return args } } else { if file_path != "" { fmt.eprintln("Unknown argument:", arg) print_help() return args } file_path = arg } } if output_flags_used > 1 { fmt.eprintln("Too many flags passed") print_help() } else { args.file_path = file_path args.type = output_flags_used == 0 ? .PlainText : output_type } return args } seek :: proc(ltx: ^Ltx) -> (rune, Ltx_Error) { next_idx := ltx.idx + 1 if next_idx >= len(ltx.source) do return 0, .EOF if current_char(ltx) == '\n' { ltx.pos = { col = 0, line = ltx.pos.line + 1, } } else { ltx.pos.col += 1 } ltx.idx = next_idx c := current_char(ltx) return c, .None } has_next :: proc(ltx: ^Ltx) -> b32 { return len(ltx.source) > ltx.idx + 1 } peek :: proc(ltx: ^Ltx) -> (rune, Ltx_Error) { if !has_next(ltx) do return 0, .EOF return rune(ltx.source[ltx.idx + 1]), .None } current_char :: proc(ltx: ^Ltx) -> rune { assert(ltx.idx < len(ltx.source), "index cannot be greater than string length") return rune(ltx.source[ltx.idx]) } consume_whitespace :: proc(ltx: ^Ltx) -> Ltx_Error { for unicode.is_white_space(current_char(ltx)) do seek(ltx) or_return // TODO: do prop error handling return .None } validate_key :: proc(key: string) -> Ltx_Error { assert(len(key) > 0, "expected non-empty key") if !unicode.is_alpha(rune(key[0])) do return .InvalidKey for c in key { if !unicode.is_alpha(c) && !unicode.is_number(c) && c != '_' do return .InvalidKey } return .None } parse :: proc(ltx: ^Ltx) -> (err: Ltx_Error) { ltx.idx = 0 stack := make([dynamic]Node) defer delete(stack) text_node_start_idx := 0 for has_next(ltx) { if current_char(ltx) == TokenArray[.Backslash] { // TODO: cehck if the char after slash is a token if text_node_start_idx < ltx.idx { node := Node { kind = .Text, text = ltx.source[text_node_start_idx:ltx.idx], } if len(stack) == 0 do append(<x.nodes, node) else do append(&stack[len(stack) - 1].children, node) } seek(ltx) or_return // skip \ tag_start_idx := ltx.idx for unicode.is_letter(current_char(ltx)) do seek(ltx) or_return assert(tag_start_idx < ltx.idx) tag_name := ltx.source[tag_start_idx:ltx.idx] node := Node { kind = .Tag, name = tag_name, } end_pos: struct { tag_name: int, fields: int, content: int, } = { tag_name = ltx.idx, fields = 0, content = 0, } consume_whitespace(ltx) or_return for current_char(ltx) == TokenArray[.LeftBracket] { c := seek(ltx) or_return field_start := ltx.idx key: string for has_next(ltx) { if c == TokenArray[.Assign] { if field_start >= ltx.idx do return .KeyExpected key = strings.trim(ltx.source[field_start:ltx.idx], " \t") if len(key) <= 0 do return .KeyExpected validate_key(key) or_return seek(ltx) or_return field_start = ltx.idx } else if c == TokenArray[.RightBracket] { if field_start >= ltx.idx do return .ValueExpected raw_value := strings.trim(ltx.source[field_start:ltx.idx], " \t") if len(raw_value) <= 0 do return .ValueExpected value := Field{} if key != "" { value.type = .Attribute value.value = raw_value } else { key = raw_value value.type = .Flag } _, _, found := map_upsert(&node.attributes, key, value) if found do return .KeyAlreadyExists seek(ltx) or_return end_pos.fields = ltx.idx break } c = seek(ltx) or_return } consume_whitespace(ltx) or_return } consume_whitespace(ltx) or_return if current_char(ltx) == TokenArray[.LeftBrace] { seek(ltx) or_return // consume { append(&stack, node) end_pos.content = ltx.idx } else { append(<x.nodes, node) } if end_pos.content != 0 do text_node_start_idx = end_pos.content else if end_pos.fields != 0 do text_node_start_idx = end_pos.fields else do text_node_start_idx = end_pos.tag_name } else if current_char(ltx) == TokenArray[.RightBrace] { if len(stack) <= 0 do return .UnexpectedRightBrace node := pop(&stack) if text_node_start_idx < ltx.idx { text_node := Node { kind = .Text, text = ltx.source[text_node_start_idx:ltx.idx], } append(&node.children, text_node) } if len(stack) > 0 { append(&stack[len(stack) - 1].children, node) } else { append(<x.nodes, node) } seek(ltx) or_return // consume } text_node_start_idx = ltx.idx } else { seek(ltx) or_return } } if text_node_start_idx < ltx.idx { append(<x.nodes, Node{kind = .Text, text = ltx.source[text_node_start_idx:ltx.idx]}) } if len(stack) > 0 do return .ClosingBraceExpected return .None } parse_file :: proc(ltx: ^Ltx, file_path: string, allocator := context.allocator) -> Ltx_Error { source, err := os.read_entire_file_from_path(file_path, allocator) if err != nil do return .CannotReadFile abs_path, abs_err := filepath.abs(file_path) if abs_err != nil do return .CannotReadFile ltx.source_path = abs_path ltx.source = string(source) return parse(ltx) } free_file :: proc(ltx: ^Ltx) -> runtime.Allocator_Error { delete(ltx.source) or_return delete(ltx.source_path) or_return return .None } print_indent :: proc(level: int) { for i in 0 ..< level { fmt.print(" ") } } print_node :: proc(node: Node, indent_level := 0) { print_indent(indent_level) switch node.kind { case .Text: fmt.printf("TEXT: \"%s\"\n", escape_white_space(node.text, context.temp_allocator)) case .Tag: fmt.print("TAG:", node.name) if len(node.attributes) > 0 { for k, v in node.attributes { if v.type == .Flag { fmt.printf(" (%s)", k) } else { fmt.printf(" {{%s: %s}}", k, v.value) } } } fmt.println() if len(node.children) > 0 { for child in node.children { print_node(child, indent_level + 1) } } } } to_text_with_smart_indent :: proc(sb: ^strings.Builder, nodes: [dynamic]Node, indent_level := 0) { should_inc_indent_level := true for node in nodes { if node.kind == .Text { should_inc_indent_level = true break } } for node in nodes { switch node.kind { case .Text: for i in 0 ..< indent_level - 1 do strings.write_rune(sb, ' ') text := strings.trim( process_white_space(node.text, context.temp_allocator), " \t\n\r\v", ) // TODO(Paul): do something about newlines; have them be limited to 2 max if they appear consecutively perhaps if text != "" { fmt.sbprintln(sb, text) } case .Tag: to_text_with_smart_indent( sb, node.children, should_inc_indent_level ? indent_level + 1 : indent_level, ) } } } // Ltx to plain text; strip ltx tags to_plain_text :: proc(sb: ^strings.Builder, nodes: [dynamic]Node) { for node in nodes { switch node.kind { case .Text: fmt.sbprint(sb, node.text) case .Tag: to_plain_text(sb, node.children) } } } // Ltx to XML // TODO: CDATA garbage to_xml :: proc(sb: ^strings.Builder, nodes: [dynamic]Node, depth := 0) { if len(nodes) <= 0 do return if depth == 0 do fmt.sbprintln(sb, "") for node in nodes { switch node.kind { case .Text: fmt.sbprint(sb, node.text) case .Tag: fmt.sbprintf(sb, "<%s", node.name) for k, v in node.attributes do fmt.sbprintf(sb, " %s=\"%s\"", k, v.value) if len(node.children) > 0 { fmt.sbprint(sb, ">") to_xml(sb, node.children, depth + 1) fmt.sbprintf(sb, "", node.name) } else { fmt.sbprint(sb, " />") } } } if depth == 0 do fmt.sbprint(sb, "\n") } // Ltx_Error to english string error_to_string :: proc(error: Ltx_Error) -> string { switch error { case .None: return "" case .EOF: return "unexpected end of file" case .KeyExpected: return "key expected before '='" case .ClosingBraceExpected: return "closing brace expected" case .ClosingBracketExpected: return "closing bracket expected" case .UnexpectedRightBrace: return "unexpected '}'" case .ValueExpected: return "value expected after '='" case .KeyAlreadyExists: return "attribute key/flag already exists in attribute" case .InvalidKey: return "invalid key" case .CannotReadFile: return "cannot read file" } return "unknown error" } // Pretty format LTX error get_error :: proc(ltx: ^Ltx, error: Ltx_Error, allocator := context.allocator) -> string { if error == .None do return "" file_path := len(ltx.source_path) > 0 ? ltx.source_path : "" line := ltx.pos.line + 1 col := ltx.pos.col + 1 error_msg := error_to_string(error) return fmt.aprintf( "%s(%d,%d): error: %s", file_path, line, col, error_msg, allocator = allocator, ) } // Escapes white space characters with in their c-string slash form escape_white_space :: proc(s: string, allocator := context.allocator) -> string { sb := strings.builder_make(allocator) for c in s { switch c { case '\t': strings.write_string(&sb, "\\t") case '\n': strings.write_string(&sb, "\\n") case '\r': strings.write_string(&sb, "\\r") case '\\': strings.write_string(&sb, "\\\\") case: strings.write_rune(&sb, c) } } return strings.to_string(sb) } // Processes and normalizes whitespace in a multi-line string. process_white_space :: proc(s: string, allocator := context.allocator) -> string { sb: strings.Builder = strings.builder_make(allocator) lines := strings.split(s, "\n") for _line in lines { line := strings.trim_right(_line, "\t ") if len(line) <= 0 do continue last_idx := 0 last_char: rune = 0 for r, i in line { if r == '\r' do continue if unicode.is_space(r) { if last_char == r { last_idx = i } else { fmt.sbprint(&sb, line[last_idx:i]) last_idx = i } } last_char = r } fmt.sbprintln(&sb, line[last_idx:]) } return strings.to_string(sb) }