Add Markdown table format support

cmd/dataxl/conversion.go

@@ -51,6 +51,8 @@ func normalizeFormat(format string) string {
 	switch format {
 	case "yml":
 		return "yaml"
+	case "md", "mdown", "mkd":
+		return "markdown"
 	case "xlsm", "xls":
 		return "xlsx"
 	default:
@@ -67,7 +69,7 @@ func inferFormat(path string) string {
 
 func validateFormat(format string) error {
 	switch format {
-	case "json", "yaml", "toml", "csv", "tsv", "xlsx":
+	case "json", "yaml", "toml", "csv", "tsv", "xlsx", "markdown":
 		return nil
 	default:
 		return fmt.Errorf("unsupported format %q", format)
@@ -75,5 +77,5 @@ func validateFormat(format string) error {
 }
 
 func isTabular(format string) bool {
-	return format == "csv" || format == "tsv" || format == "xlsx"
+	return format == "csv" || format == "tsv" || format == "xlsx" || format == "markdown"
 }

cmd/dataxl/conversion_matrix_test.go

@@ -6,7 +6,7 @@ import (
 )
 
 func TestConversionMatrixPreservesRepresentativeTable(t *testing.T) {
-	formats := []string{"json", "yaml", "toml", "csv", "tsv", "xlsx"}
+	formats := []string{"json", "yaml", "toml", "csv", "tsv", "xlsx", "markdown"}
 	records := []any{
 		map[string]any{
 			"active": true,

cmd/dataxl/data_integrity_test.go

@@ -220,7 +220,7 @@ func TestDelimitedSingleEmptyCellRowRoundTrip(t *testing.T) {
 
 func TestStructuredRecordsWithoutScalarFieldsAreRejectedForTables(t *testing.T) {
 	for _, input := range []string{`{}`, `[{}]`} {
-		for _, format := range []string{"csv", "tsv", "xlsx"} {
+		for _, format := range []string{"csv", "tsv", "xlsx", "markdown"} {
 			t.Run(format+"_"+input, func(t *testing.T) {
 				_, err := convert([]byte(input), "json", format, "Sheet1", false)
 				if err == nil || !strings.Contains(err.Error(), "no scalar fields") {
@@ -232,7 +232,7 @@ func TestStructuredRecordsWithoutScalarFieldsAreRejectedForTables(t *testing.T)
 }
 
 func TestEmptyRecordListCanRoundTripThroughDelimitedFormats(t *testing.T) {
-	for _, format := range []string{"csv", "tsv"} {
+	for _, format := range []string{"csv", "tsv", "markdown"} {
 		t.Run(format, func(t *testing.T) {
 			tabular, err := convert([]byte(`[]`), "json", format, "Sheet1", false)
 			if err != nil {

cmd/dataxl/doc.go

@@ -1,7 +1,7 @@
 // Package main implements the dataxl command-line converter.
 //
 // Conversion uses two internal representations: structured Go values for
-// JSON/YAML/TOML and table for CSV/TSV/XLSX. Nested structured values cross the
+// JSON/YAML/TOML and table for CSV/TSV/XLSX/Markdown. Nested structured values cross the
 // boundary through dotted and indexed column paths such as user.name and
 // items[0].sku.
 package main

cmd/dataxl/main.go

@@ -60,8 +60,8 @@ func parseOptions(args []string, stderr io.Writer) (options, error) {
 	fs.SetOutput(stderr)
 	fs.StringVar(&opt.inFile, "i", "", "input file, defaults to stdin")
 	fs.StringVar(&opt.outFile, "o", "", "output file, defaults to stdout")
-	fs.StringVar(&opt.from, "from", "", "input format: json, yaml, toml, csv, tsv, xlsx")
-	fs.StringVar(&opt.to, "to", "", "output format: json, yaml, toml, csv, tsv, xlsx")
+	fs.StringVar(&opt.from, "from", "", "input format: json, yaml, toml, csv, tsv, xlsx, markdown")
+	fs.StringVar(&opt.to, "to", "", "output format: json, yaml, toml, csv, tsv, xlsx, markdown")
 	fs.StringVar(&opt.sheet, "sheet", "Sheet1", "worksheet name for xlsx input/output")
 	fs.BoolVar(&opt.pretty, "pretty", true, "pretty-print structured output")
 	fs.BoolVar(&opt.version, "version", false, "print version and exit")
@@ -72,7 +72,7 @@ func parseOptions(args []string, stderr io.Writer) (options, error) {
   dataxl -from json -to xlsx -i data.json -o data.xlsx
 
 Formats:
-  json, yaml/yml, toml, csv, tsv, xlsx
+  json, yaml/yml, toml, csv, tsv, xlsx, markdown/md
 
 Notes:
   Structured records are flattened into spreadsheet columns such as user.name

cmd/dataxl/main_test.go

@@ -129,6 +129,49 @@ func TestResolveFormatsFromFileExtensions(t *testing.T) {
 	}
 }
 
+func TestResolveMarkdownFormatFromFileExtension(t *testing.T) {
+	opt := options{inFile: "input.json", outFile: "output.md"}
+	if err := opt.resolveFormats(); err != nil {
+		t.Fatal(err)
+	}
+	if opt.from != "json" || opt.to != "markdown" {
+		t.Fatalf("resolved formats = %q -> %q, want json -> markdown", opt.from, opt.to)
+	}
+}
+
+func TestMarkdownTableRoundTrip(t *testing.T) {
+	want := table{
+		Header: []string{"id", "name", "note"},
+		Rows:   [][]string{{"1", "Alice | Bob", "line 1\nline 2\\n"}},
+	}
+	encoded, err := encodeTable(want, "markdown", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := parseTable(encoded, "markdown", "")
+	if err != nil {
+		t.Fatalf("parse generated Markdown: %v\n%s", err, encoded)
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("table = %#v, want %#v\n%s", got, want, encoded)
+	}
+}
+
+func TestMarkdownTableAcceptsAlignmentAndOptionalOuterPipes(t *testing.T) {
+	input := []byte("id | name | note\n---: | :--- | :---:\n1 | Alice \\| Bob | ok\n")
+	got, err := parseTable(input, "markdown", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := table{
+		Header: []string{"id", "name", "note"},
+		Rows:   [][]string{{"1", "Alice | Bob", "ok"}},
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("table = %#v, want %#v", got, want)
+	}
+}
+
 func TestVersionFlagDoesNotRequireFormats(t *testing.T) {
 	oldVersion := buildVersion
 	buildVersion = "v9.8.7-test"

cmd/dataxl/table.go

@@ -5,14 +5,16 @@ import (
 	"encoding/csv"
 	"encoding/json"
 	"fmt"
+	"html"
 	"sort"
 	"strconv"
+	"strings"
 	"unicode/utf8"
 
 	"github.com/xuri/excelize/v2"
 )
 
-// table is the common representation for CSV, TSV and XLSX. Rows are padded
+// table is the common representation for CSV, TSV, XLSX and Markdown. Rows are padded
 // to Header width when read, which keeps subsequent conversions rectangular.
 type table struct {
 	Header []string
@@ -36,6 +38,8 @@ func parseTable(input []byte, format, sheet string) (table, error) {
 			return table{}, err
 		}
 		return rowsToTable(rows)
+	case "markdown":
+		return readMarkdown(input)
 	default:
 		return table{}, fmt.Errorf("format %q is not tabular", format)
 	}
@@ -93,11 +97,206 @@ func encodeTable(t table, format, sheet string) ([]byte, error) {
 		return writeDelimited(t, '\t')
 	case "xlsx":
 		return writeXLSX(t, sheet)
+	case "markdown":
+		return writeMarkdown(t)
 	default:
 		return nil, fmt.Errorf("format %q is not tabular", format)
 	}
 }
 
+func readMarkdown(input []byte) (table, error) {
+	input = bytes.TrimPrefix(input, []byte{0xEF, 0xBB, 0xBF})
+	if !utf8.Valid(input) {
+		return table{}, fmt.Errorf("markdown input is not valid UTF-8")
+	}
+	text := strings.ReplaceAll(string(input), "\r\n", "\n")
+	text = strings.TrimSpace(text)
+	if text == "" {
+		return table{}, nil
+	}
+	lines := strings.Split(text, "\n")
+	if len(lines) < 2 {
+		return table{}, fmt.Errorf("markdown table requires a header and separator row")
+	}
+	header, err := parseMarkdownRow(lines[0])
+	if err != nil {
+		return table{}, fmt.Errorf("markdown header: %w", err)
+	}
+	separator, err := parseMarkdownRow(lines[1])
+	if err != nil {
+		return table{}, fmt.Errorf("markdown separator: %w", err)
+	}
+	if len(separator) != len(header) {
+		return table{}, fmt.Errorf("markdown separator has %d fields but the header has %d", len(separator), len(header))
+	}
+	for i, cell := range separator {
+		value := strings.TrimSpace(cell)
+		value = strings.TrimPrefix(value, ":")
+		value = strings.TrimSuffix(value, ":")
+		if len(value) < 3 || strings.Trim(value, "-") != "" {
+			return table{}, fmt.Errorf("markdown separator field %d is invalid", i+1)
+		}
+	}
+	rows := make([][]string, 0, len(lines)-2)
+	for i, line := range lines[2:] {
+		if strings.TrimSpace(line) == "" {
+			continue
+		}
+		row, err := parseMarkdownRow(line)
+		if err != nil {
+			return table{}, fmt.Errorf("markdown row %d: %w", i+3, err)
+		}
+		if len(row) > len(header) {
+			return table{}, fmt.Errorf("row %d has %d fields but the header has %d", i+3, len(row), len(header))
+		}
+		rows = append(rows, padRow(row, len(header)))
+	}
+	return table{Header: header, Rows: rows}, nil
+}
+
+func parseMarkdownRow(line string) ([]string, error) {
+	line = strings.TrimSpace(strings.TrimSuffix(line, "\r"))
+	if strings.HasPrefix(line, "|") {
+		line = line[1:]
+	}
+	if hasUnescapedTrailingPipe(line) {
+		line = strings.TrimSpace(line[:len(line)-1])
+	}
+	var cells []string
+	var cell strings.Builder
+	escaped := false
+	for _, r := range line {
+		if escaped {
+			cell.WriteRune('\\')
+			cell.WriteRune(r)
+			escaped = false
+			continue
+		}
+		if r == '\\' {
+			escaped = true
+			continue
+		}
+		if r == '|' {
+			cells = append(cells, decodeMarkdownCell(strings.Trim(cell.String(), " \t")))
+			cell.Reset()
+			continue
+		}
+		cell.WriteRune(r)
+	}
+	if escaped {
+		return nil, fmt.Errorf("row ends with an incomplete escape")
+	}
+	cells = append(cells, decodeMarkdownCell(strings.Trim(cell.String(), " \t")))
+	return cells, nil
+}
+
+func decodeMarkdownCell(value string) string {
+	var b strings.Builder
+	escaped := false
+	for _, r := range value {
+		if escaped {
+			if r == '\\' || r == '|' {
+				b.WriteRune(r)
+			} else {
+				b.WriteRune('\\')
+				b.WriteRune(r)
+			}
+			escaped = false
+			continue
+		}
+		if r == '\\' {
+			escaped = true
+			continue
+		}
+		b.WriteRune(r)
+	}
+	return html.UnescapeString(b.String())
+}
+
+func hasUnescapedTrailingPipe(line string) bool {
+	line = strings.TrimSpace(line)
+	if !strings.HasSuffix(line, "|") {
+		return false
+	}
+	backslashes := 0
+	for i := len(line) - 2; i >= 0 && line[i] == '\\'; i-- {
+		backslashes++
+	}
+	return backslashes%2 == 0
+}
+
+func writeMarkdown(t table) ([]byte, error) {
+	if len(t.Header) == 0 && len(t.Rows) == 0 {
+		return nil, nil
+	}
+	if len(t.Header) == 0 {
+		return nil, fmt.Errorf("markdown table has rows but no header")
+	}
+	var b strings.Builder
+	writeRow := func(row []string) {
+		b.WriteString("| ")
+		for i, value := range row {
+			if i > 0 {
+				b.WriteString(" | ")
+			}
+			b.WriteString(escapeMarkdownCell(value))
+		}
+		b.WriteString(" |\n")
+	}
+	writeRow(t.Header)
+	b.WriteString("|")
+	for range t.Header {
+		b.WriteString(" --- |")
+	}
+	b.WriteByte('\n')
+	for _, row := range t.Rows {
+		writeRow(padRow(row, len(t.Header)))
+	}
+	return []byte(b.String()), nil
+}
+
+func escapeMarkdownCell(value string) string {
+	leadingEnd := len(value) - len(strings.TrimLeft(value, " \t"))
+	trailingStart := len(strings.TrimRight(value, " \t"))
+	var b strings.Builder
+	for i, r := range value {
+		switch r {
+		case '\\':
+			b.WriteString("\\\\")
+		case '|':
+			b.WriteString("\\|")
+		case '\r':
+			b.WriteString("&#13;")
+		case '\n':
+			b.WriteString("&#10;")
+		case ' ':
+			if i < leadingEnd || i >= trailingStart {
+				b.WriteString("&#32;")
+			} else {
+				b.WriteByte(' ')
+			}
+		case '\t':
+			if i < leadingEnd || i >= trailingStart {
+				b.WriteString("&#9;")
+			} else {
+				b.WriteByte('\t')
+			}
+		default:
+			switch r {
+			case '&':
+				b.WriteString("&amp;")
+			case '<':
+				b.WriteString("&lt;")
+			case '>':
+				b.WriteString("&gt;")
+			default:
+				b.WriteRune(r)
+			}
+		}
+	}
+	return b.String()
+}
+
 func writeDelimited(t table, comma rune) ([]byte, error) {
 	var b bytes.Buffer
 	w := csv.NewWriter(&b)