From 3ea4d31fd0a690a35b65a821a070a67df7f4ed30 Mon Sep 17 00:00:00 2001 From: Luno Date: Sat, 18 Jul 2026 16:25:50 +0900 Subject: [PATCH] Refactor conversion pipeline by responsibility --- cmd/dataxl/conversion.go | 71 +++++ cmd/dataxl/doc.go | 7 + cmd/dataxl/main.go | 558 ++------------------------------------- cmd/dataxl/main_test.go | 71 +++++ cmd/dataxl/path.go | 134 ++++++++++ cmd/dataxl/structured.go | 85 ++++++ cmd/dataxl/table.go | 265 +++++++++++++++++++ 7 files changed, 662 insertions(+), 529 deletions(-) create mode 100644 cmd/dataxl/conversion.go create mode 100644 cmd/dataxl/doc.go create mode 100644 cmd/dataxl/path.go create mode 100644 cmd/dataxl/structured.go create mode 100644 cmd/dataxl/table.go diff --git a/cmd/dataxl/conversion.go b/cmd/dataxl/conversion.go new file mode 100644 index 0000000..7514300 --- /dev/null +++ b/cmd/dataxl/conversion.go @@ -0,0 +1,71 @@ +package main + +import ( + "fmt" + "path/filepath" + "strings" +) + +// convert routes data through either the structured-value or table internal +// representation. Keeping this matrix in one place makes format adapters +// independent from CLI and file I/O concerns. +func convert(input []byte, from, to, sheet string, pretty bool) ([]byte, error) { + switch { + case isTabular(from) && isTabular(to): + t, err := parseTable(input, from, sheet) + if err != nil { + return nil, err + } + return encodeTable(t, to, sheet) + case isTabular(from): + t, err := parseTable(input, from, sheet) + if err != nil { + return nil, err + } + return encodeStructured(tableToRecords(t), to, pretty) + case isTabular(to): + value, err := parseStructured(input, from) + if err != nil { + return nil, err + } + return encodeTable(valueToTable(value), to, sheet) + default: + value, err := parseStructured(input, from) + if err != nil { + return nil, err + } + return encodeStructured(value, to, pretty) + } +} + +func normalizeFormat(format string) string { + format = strings.ToLower(strings.TrimPrefix(strings.TrimSpace(format), ".")) + switch format { + case "yml": + return "yaml" + case "xlsm", "xls": + return "xlsx" + default: + return format + } +} + +func inferFormat(path string) string { + if path == "" || path == "-" { + return "" + } + return normalizeFormat(filepath.Ext(path)) +} + +func validateFormat(format string) error { + switch format { + case "json", "yaml", "toml", "csv", "tsv", "xlsx": + return nil + default: + return fmt.Errorf("unsupported format %q", format) + } +} + +func isTabular(format string) bool { + return format == "csv" || format == "tsv" || format == "xlsx" +} diff --git a/cmd/dataxl/doc.go b/cmd/dataxl/doc.go new file mode 100644 index 0000000..64d1edb --- /dev/null +++ b/cmd/dataxl/doc.go @@ -0,0 +1,7 @@ +// Package main implements the dataxl command-line converter. +// +// Conversion uses two internal representations: structured Go values for +// JSON/YAML/TOML and table for CSV/TSV/XLSX. Nested structured values cross the +// boundary through dotted and indexed column paths such as user.name and +// items[0].sku. +package main diff --git a/cmd/dataxl/main.go b/cmd/dataxl/main.go index 9d39a09..cb46e84 100644 --- a/cmd/dataxl/main.go +++ b/cmd/dataxl/main.go @@ -1,25 +1,15 @@ package main import ( - "bytes" - "encoding/csv" - "encoding/json" - "errors" "flag" "fmt" "io" "os" - "path/filepath" - "regexp" - "sort" - "strconv" "strings" - - "github.com/BurntSushi/toml" - "github.com/xuri/excelize/v2" - "gopkg.in/yaml.v3" ) +// options contains CLI concerns only. Conversion functions receive explicit +// arguments so they can be reused and tested without constructing a FlagSet. type options struct { inFile string outFile string @@ -29,11 +19,6 @@ type options struct { pretty bool } -type table struct { - Header []string - Rows [][]string -} - func main() { if err := run(os.Args[1:], os.Stdin, os.Stdout, os.Stderr); err != nil { fmt.Fprintln(os.Stderr, "dataxl:", err) @@ -42,6 +27,26 @@ func main() { } func run(args []string, stdin io.Reader, stdout, stderr io.Writer) error { + opt, err := parseOptions(args, stderr) + if err != nil { + return err + } + if err := opt.resolveFormats(); err != nil { + return err + } + + input, err := readInput(opt.inFile, stdin) + if err != nil { + return err + } + output, err := convert(input, opt.from, opt.to, opt.sheet, opt.pretty) + if err != nil { + return err + } + return writeOutput(opt.outFile, stdout, output) +} + +func parseOptions(args []string, stderr io.Writer) (options, error) { var opt options fs := flag.NewFlagSet("dataxl", flag.ContinueOnError) fs.SetOutput(stderr) @@ -66,16 +71,15 @@ Notes: converting back to json/yaml/toml.`) } if err := fs.Parse(args); err != nil { - return err + return options{}, err } if fs.NArg() != 0 { - return fmt.Errorf("unexpected arguments: %s", strings.Join(fs.Args(), " ")) + return options{}, fmt.Errorf("unexpected arguments: %s", strings.Join(fs.Args(), " ")) } + return opt, nil +} - input, err := readInput(opt.inFile, stdin) - if err != nil { - return err - } +func (opt *options) resolveFormats() error { opt.from = normalizeFormat(opt.from) opt.to = normalizeFormat(opt.to) if opt.from == "" { @@ -85,60 +89,12 @@ Notes: opt.to = inferFormat(opt.outFile) } if opt.from == "" || opt.to == "" { - return errors.New("both -from and -to are required when a format cannot be inferred from file names") + return fmt.Errorf("both -from and -to are required when a format cannot be inferred from file names") } if err := validateFormat(opt.from); err != nil { return err } - if err := validateFormat(opt.to); err != nil { - return err - } - - var out []byte - // Use table as the common representation whenever either side is a - // spreadsheet-like format. Structured formats can then share the same - // flatten/restore path for CSV, TSV, and XLSX. - if isTabular(opt.from) && isTabular(opt.to) { - t, err := parseTable(input, opt.from, opt.sheet) - if err != nil { - return err - } - out, err = encodeTable(t, opt.to, opt.sheet) - if err != nil { - return err - } - } else if isTabular(opt.from) { - t, err := parseTable(input, opt.from, opt.sheet) - if err != nil { - return err - } - records := tableToRecords(t) - out, err = encodeStructured(records, opt.to, opt.pretty) - if err != nil { - return err - } - } else if isTabular(opt.to) { - value, err := parseStructured(input, opt.from) - if err != nil { - return err - } - t := valueToTable(value) - out, err = encodeTable(t, opt.to, opt.sheet) - if err != nil { - return err - } - } else { - value, err := parseStructured(input, opt.from) - if err != nil { - return err - } - out, err = encodeStructured(value, opt.to, opt.pretty) - if err != nil { - return err - } - } - - return writeOutput(opt.outFile, stdout, out) + return validateFormat(opt.to) } func readInput(path string, stdin io.Reader) ([]byte, error) { @@ -155,459 +111,3 @@ func writeOutput(path string, stdout io.Writer, data []byte) error { } return os.WriteFile(path, data, 0o644) } - -func normalizeFormat(format string) string { - format = strings.ToLower(strings.TrimPrefix(strings.TrimSpace(format), ".")) - switch format { - case "yml": - return "yaml" - case "xlsm", "xls": - return "xlsx" - default: - return format - } -} - -func inferFormat(path string) string { - if path == "" || path == "-" { - return "" - } - return normalizeFormat(strings.TrimPrefix(filepath.Ext(path), ".")) -} - -func validateFormat(format string) error { - switch format { - case "json", "yaml", "toml", "csv", "tsv", "xlsx": - return nil - default: - return fmt.Errorf("unsupported format %q", format) - } -} - -func isTabular(format string) bool { - return format == "csv" || format == "tsv" || format == "xlsx" -} - -func parseStructured(input []byte, format string) (any, error) { - var value any - switch format { - case "json": - dec := json.NewDecoder(bytes.NewReader(input)) - dec.UseNumber() - if err := dec.Decode(&value); err != nil { - return nil, err - } - case "yaml": - if err := yaml.Unmarshal(input, &value); err != nil { - return nil, err - } - value = normalizeYAML(value) - case "toml": - var m map[string]any - if err := toml.Unmarshal(input, &m); err != nil { - return nil, err - } - value = m - default: - return nil, fmt.Errorf("format %q is not structured", format) - } - return value, nil -} - -func encodeStructured(value any, format string, pretty bool) ([]byte, error) { - switch format { - case "json": - if pretty { - return json.MarshalIndent(value, "", " ") - } - return json.Marshal(value) - case "yaml": - return yaml.Marshal(value) - case "toml": - m, ok := value.(map[string]any) - if !ok { - m = map[string]any{"rows": value} - } - var b bytes.Buffer - err := toml.NewEncoder(&b).Encode(m) - return b.Bytes(), err - default: - return nil, fmt.Errorf("format %q is not structured", format) - } -} - -func normalizeYAML(value any) any { - switch v := value.(type) { - case map[string]any: - out := make(map[string]any, len(v)) - for key, child := range v { - out[key] = normalizeYAML(child) - } - return out - case map[any]any: - out := make(map[string]any, len(v)) - for key, child := range v { - out[fmt.Sprint(key)] = normalizeYAML(child) - } - return out - case []any: - for i := range v { - v[i] = normalizeYAML(v[i]) - } - } - return value -} - -func parseTable(input []byte, format, sheet string) (table, error) { - switch format { - case "csv": - return readDelimited(input, ',') - case "tsv": - return readDelimited(input, '\t') - case "xlsx": - f, err := excelize.OpenReader(bytes.NewReader(input)) - if err != nil { - return table{}, err - } - defer f.Close() - rows, err := f.GetRows(sheet) - if err != nil { - return table{}, err - } - return rowsToTable(rows), nil - default: - return table{}, fmt.Errorf("format %q is not tabular", format) - } -} - -func readDelimited(input []byte, comma rune) (table, error) { - r := csv.NewReader(bytes.NewReader(input)) - r.Comma = comma - r.FieldsPerRecord = -1 - r.TrimLeadingSpace = comma == ',' - rows, err := r.ReadAll() - if err != nil { - return table{}, err - } - return rowsToTable(rows), nil -} - -func rowsToTable(rows [][]string) table { - if len(rows) == 0 { - return table{} - } - width := 0 - for _, row := range rows { - if len(row) > width { - width = len(row) - } - } - header := padRow(rows[0], width) - body := make([][]string, 0, len(rows)-1) - for _, row := range rows[1:] { - body = append(body, padRow(row, width)) - } - return table{Header: header, Rows: body} -} - -func padRow(row []string, width int) []string { - out := make([]string, width) - copy(out, row) - return out -} - -func encodeTable(t table, format, sheet string) ([]byte, error) { - switch format { - case "csv": - return writeDelimited(t, ',') - case "tsv": - return writeDelimited(t, '\t') - case "xlsx": - return writeXLSX(t, sheet) - default: - return nil, fmt.Errorf("format %q is not tabular", format) - } -} - -func writeDelimited(t table, comma rune) ([]byte, error) { - var b bytes.Buffer - w := csv.NewWriter(&b) - w.Comma = comma - if err := w.Write(t.Header); err != nil { - return nil, err - } - for _, row := range t.Rows { - if err := w.Write(row); err != nil { - return nil, err - } - } - w.Flush() - return b.Bytes(), w.Error() -} - -func writeXLSX(t table, sheet string) ([]byte, error) { - f := excelize.NewFile() - defaultSheet := f.GetSheetName(0) - if sheet == "" { - sheet = "Sheet1" - } - if defaultSheet != sheet { - if err := f.SetSheetName(defaultSheet, sheet); err != nil { - return nil, err - } - } - rows := append([][]string{t.Header}, t.Rows...) - for r, row := range rows { - for c, value := range row { - cell, err := excelize.CoordinatesToCellName(c+1, r+1) - if err != nil { - return nil, err - } - if err := f.SetCellValue(sheet, cell, value); err != nil { - return nil, err - } - } - } - if len(t.Header) > 0 { - // The generated workbook is meant for editing, so keep the header row - // visible and visually distinct. - end, _ := excelize.CoordinatesToCellName(len(t.Header), 1) - style, _ := f.NewStyle(&excelize.Style{Font: &excelize.Font{Bold: true}}) - _ = f.SetCellStyle(sheet, "A1", end, style) - _ = f.SetPanes(sheet, &excelize.Panes{ - Freeze: true, - Split: false, - XSplit: 0, - YSplit: 1, - TopLeftCell: "A2", - ActivePane: "bottomLeft", - }) - } - var b bytes.Buffer - if err := f.Write(&b); err != nil { - return nil, err - } - return b.Bytes(), nil -} - -func valueToTable(value any) table { - records := recordsFromValue(value) - flatRows := make([]map[string]string, 0, len(records)) - seen := map[string]bool{} - var header []string - for _, record := range records { - flat := map[string]string{} - flatten("", record, flat) - for key := range flat { - if !seen[key] { - seen[key] = true - header = append(header, key) - } - } - flatRows = append(flatRows, flat) - } - // Stable column ordering keeps generated CSV/TSV/XLSX diffs predictable. - sort.Strings(header) - rows := make([][]string, 0, len(flatRows)) - for _, flat := range flatRows { - row := make([]string, len(header)) - for i, key := range header { - row[i] = flat[key] - } - rows = append(rows, row) - } - return table{Header: header, Rows: rows} -} - -func recordsFromValue(value any) []any { - switch v := value.(type) { - case []any: - return v - case map[string]any: - // Common wrapper keys let TOML and object-shaped inputs represent a - // table without adding a format-specific flag. - for _, key := range []string{"rows", "records", "items"} { - if rows, ok := v[key].([]any); ok { - return rows - } - } - return []any{v} - default: - return []any{v} - } -} - -// flatten converts nested values into spreadsheet-safe column paths such as -// user.name and items[0].sku. -func flatten(prefix string, value any, out map[string]string) { - switch v := value.(type) { - case map[string]any: - if len(v) == 0 && prefix != "" { - out[prefix] = "{}" - return - } - keys := make([]string, 0, len(v)) - for key := range v { - keys = append(keys, key) - } - sort.Strings(keys) - for _, key := range keys { - childPrefix := key - if prefix != "" { - childPrefix = prefix + "." + key - } - flatten(childPrefix, v[key], out) - } - case []any: - if len(v) == 0 && prefix != "" { - out[prefix] = "[]" - return - } - for i, child := range v { - flatten(fmt.Sprintf("%s[%d]", prefix, i), child, out) - } - default: - if prefix == "" { - prefix = "value" - } - out[prefix] = scalarString(v) - } -} - -func scalarString(value any) string { - switch v := value.(type) { - case nil: - return "" - case json.Number: - return v.String() - case string: - return v - case bool: - return strconv.FormatBool(v) - case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64: - return fmt.Sprint(v) - default: - data, err := json.Marshal(v) - if err != nil { - return fmt.Sprint(v) - } - return string(data) - } -} - -// tableToRecords restores each row by interpreting header cells as path -// expressions. Empty headers are ignored so spare spreadsheet columns are safe. -func tableToRecords(t table) []map[string]any { - var records []map[string]any - for _, row := range t.Rows { - record := map[string]any{} - for i, header := range t.Header { - header = strings.TrimSpace(header) - if header == "" || i >= len(row) { - continue - } - setPath(record, header, parseCell(row[i])) - } - records = append(records, record) - } - return records -} - -// parseCell keeps spreadsheet round trips useful while avoiding broad type -// inference that could surprise users editing IDs or codes. -func parseCell(s string) any { - s = strings.TrimSpace(s) - if s == "" { - return "" - } - if s == "true" { - return true - } - if s == "false" { - return false - } - if i, err := strconv.ParseInt(s, 10, 64); err == nil { - return i - } - if f, err := strconv.ParseFloat(s, 64); err == nil && strings.ContainsAny(s, ".eE") { - return f - } - return s -} - -var pathTokenRE = regexp.MustCompile(`([^\.\[\]]+)|\[(\d+)\]`) - -// setPath creates maps and slices as needed for dotted and indexed header -// paths. Invalid intermediate shapes are left unchanged instead of guessing. -func setPath(root map[string]any, path string, value any) { - tokens := parsePath(path) - if len(tokens) == 0 { - return - } - var cur any = root - for i, token := range tokens { - last := i == len(tokens)-1 - nextIsIndex := !last && tokens[i+1].isIndex - if token.isIndex { - continue - } - m, ok := cur.(map[string]any) - if !ok { - return - } - if last { - m[token.key] = value - return - } - if _, ok := m[token.key]; !ok { - if nextIsIndex { - m[token.key] = []any{} - } else { - m[token.key] = map[string]any{} - } - } - if nextIsIndex { - slice, _ := m[token.key].([]any) - index := tokens[i+1].index - for len(slice) <= index { - if i+2 < len(tokens) && !tokens[i+2].isIndex { - slice = append(slice, map[string]any{}) - } else { - slice = append(slice, nil) - } - } - m[token.key] = slice - if i+2 == len(tokens) { - slice[index] = value - return - } - if slice[index] == nil { - slice[index] = map[string]any{} - } - cur = slice[index] - i++ - continue - } - cur = m[token.key] - } -} - -type pathToken struct { - key string - index int - isIndex bool -} - -func parsePath(path string) []pathToken { - matches := pathTokenRE.FindAllStringSubmatch(path, -1) - tokens := make([]pathToken, 0, len(matches)) - for _, m := range matches { - if m[1] != "" { - tokens = append(tokens, pathToken{key: m[1]}) - continue - } - index, _ := strconv.Atoi(m[2]) - tokens = append(tokens, pathToken{index: index, isIndex: true}) - } - return tokens -} diff --git a/cmd/dataxl/main_test.go b/cmd/dataxl/main_test.go index 5787241..7982dbf 100644 --- a/cmd/dataxl/main_test.go +++ b/cmd/dataxl/main_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "os" "path/filepath" + "reflect" "strings" "testing" ) @@ -88,6 +89,76 @@ func TestJSONToXLSXAndBack(t *testing.T) { } } +func TestResolveFormatsFromFileExtensions(t *testing.T) { + opt := options{inFile: "input.YML", outFile: "output.XLSM"} + if err := opt.resolveFormats(); err != nil { + t.Fatal(err) + } + if opt.from != "yaml" || opt.to != "xlsx" { + t.Fatalf("resolved formats = %q -> %q, want yaml -> xlsx", opt.from, opt.to) + } +} + +func TestConvertStructuredToStructured(t *testing.T) { + got, err := convert([]byte("name: Alice\nactive: true\n"), "yaml", "json", "Sheet1", false) + if err != nil { + t.Fatal(err) + } + if string(got) != `{"active":true,"name":"Alice"}` { + t.Fatalf("JSON = %s", got) + } +} + +func TestRowsToTablePadsRaggedRows(t *testing.T) { + got := rowsToTable([][]string{{"a", "b"}, {"1"}, {"2", "3", "4"}}) + want := table{ + Header: []string{"a", "b", ""}, + Rows: [][]string{{"1", "", ""}, {"2", "3", "4"}}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("table = %#v, want %#v", got, want) + } +} + +func TestParseCellConservativeInference(t *testing.T) { + tests := map[string]any{ + "": "", + "true": true, + "42": int64(42), + "3.14": 3.14, + "00123": "00123", + "1e3": 1000.0, + "Alice": "Alice", + } + for input, want := range tests { + if got := parseCell(input); !reflect.DeepEqual(got, want) { + t.Errorf("parseCell(%q) = %#v, want %#v", input, got, want) + } + } +} + +func TestSetPathDoesNotOverwriteConflictingShape(t *testing.T) { + record := map[string]any{} + setPath(record, "user", "Alice") + setPath(record, "user.name", "Bob") + if got := record["user"]; got != "Alice" { + t.Fatalf("user = %#v, want original scalar", got) + } +} + +func TestSetPathRestoresNestedArrays(t *testing.T) { + record := map[string]any{} + setPath(record, "orders[0].items[1].sku", "B-002") + + orders := record["orders"].([]any) + order := orders[0].(map[string]any) + items := order["items"].([]any) + item := items[1].(map[string]any) + if item["sku"] != "B-002" { + t.Fatalf("nested sku = %#v, want B-002", item["sku"]) + } +} + type ioDiscard struct{} func (ioDiscard) Write(p []byte) (int, error) { return len(p), nil } diff --git a/cmd/dataxl/path.go b/cmd/dataxl/path.go new file mode 100644 index 0000000..ee589f5 --- /dev/null +++ b/cmd/dataxl/path.go @@ -0,0 +1,134 @@ +package main + +import ( + "regexp" + "strconv" + "strings" +) + +// tableToRecords interprets non-empty headers as path expressions. Blank +// headers are intentionally ignored so spare spreadsheet columns are harmless. +func tableToRecords(t table) []map[string]any { + records := make([]map[string]any, 0, len(t.Rows)) + for _, row := range t.Rows { + record := make(map[string]any) + for i, header := range t.Header { + header = strings.TrimSpace(header) + if header == "" || i >= len(row) { + continue + } + setPath(record, header, parseCell(row[i])) + } + records = append(records, record) + } + return records +} + +// parseCell deliberately performs narrow inference. In particular, strings +// such as 00123 remain strings because converting identifiers is surprising. +func parseCell(s string) any { + s = strings.TrimSpace(s) + if s == "" { + return "" + } + if s == "true" { + return true + } + if s == "false" { + return false + } + // Preserve zero-padded identifiers such as postal codes and product codes. + leadingZero := len(s) > 1 && s[0] == '0' + negativeLeadingZero := len(s) > 2 && s[0] == '-' && s[1] == '0' + if i, err := strconv.ParseInt(s, 10, 64); err == nil && !leadingZero && !negativeLeadingZero { + return i + } + if f, err := strconv.ParseFloat(s, 64); err == nil && strings.ContainsAny(s, ".eE") { + return f + } + return s +} + +var pathTokenRE = regexp.MustCompile(`([^\.\[\]]+)|\[(\d+)\]`) + +type pathToken struct { + key string + index int + isIndex bool +} + +func parsePath(path string) []pathToken { + matches := pathTokenRE.FindAllStringSubmatch(path, -1) + tokens := make([]pathToken, 0, len(matches)) + for _, match := range matches { + if match[1] != "" { + tokens = append(tokens, pathToken{key: match[1]}) + continue + } + index, _ := strconv.Atoi(match[2]) // regexp guarantees decimal digits. + tokens = append(tokens, pathToken{index: index, isIndex: true}) + } + return tokens +} + +// setPath creates maps and slices while walking a dotted/indexed header. The +// supported grammar is a sequence of map keys with optional array indices, +// e.g. "orders[0].items[1].sku". Conflicting intermediate shapes are left +// unchanged rather than silently overwriting data from an earlier column. +func setPath(root map[string]any, path string, value any) { + tokens := parsePath(path) + if len(tokens) == 0 { + return + } + var current any = root + for i := 0; i < len(tokens); i++ { + token := tokens[i] + last := i == len(tokens)-1 + nextIsIndex := !last && tokens[i+1].isIndex + if token.isIndex { + continue + } + m, ok := current.(map[string]any) + if !ok { + return + } + if last { + m[token.key] = value + return + } + if _, exists := m[token.key]; !exists { + if nextIsIndex { + m[token.key] = []any{} + } else { + m[token.key] = map[string]any{} + } + } + if !nextIsIndex { + current = m[token.key] + continue + } + + slice, ok := m[token.key].([]any) + if !ok { + return + } + index := tokens[i+1].index + for len(slice) <= index { + if i+2 < len(tokens) && !tokens[i+2].isIndex { + slice = append(slice, map[string]any{}) + } else { + slice = append(slice, nil) + } + } + m[token.key] = slice + if i+2 == len(tokens) { + slice[index] = value + return + } + if slice[index] == nil { + slice[index] = map[string]any{} + } + current = slice[index] + i++ // The array index was consumed together with its key. + } +} diff --git a/cmd/dataxl/structured.go b/cmd/dataxl/structured.go new file mode 100644 index 0000000..fc50c2d --- /dev/null +++ b/cmd/dataxl/structured.go @@ -0,0 +1,85 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/BurntSushi/toml" + "gopkg.in/yaml.v3" +) + +// parseStructured decodes a format into maps, slices and scalar Go values. +// json.Number is retained so large JSON integers do not pass through float64. +func parseStructured(input []byte, format string) (any, error) { + var value any + switch format { + case "json": + dec := json.NewDecoder(bytes.NewReader(input)) + dec.UseNumber() + if err := dec.Decode(&value); err != nil { + return nil, err + } + case "yaml": + if err := yaml.Unmarshal(input, &value); err != nil { + return nil, err + } + value = normalizeYAML(value) + case "toml": + var m map[string]any + if err := toml.Unmarshal(input, &m); err != nil { + return nil, err + } + value = m + default: + return nil, fmt.Errorf("format %q is not structured", format) + } + return value, nil +} + +func encodeStructured(value any, format string, pretty bool) ([]byte, error) { + switch format { + case "json": + if pretty { + return json.MarshalIndent(value, "", " ") + } + return json.Marshal(value) + case "yaml": + return yaml.Marshal(value) + case "toml": + m, ok := value.(map[string]any) + if !ok { + // TOML has no top-level array, so preserve it under a documented key. + m = map[string]any{"rows": value} + } + var b bytes.Buffer + err := toml.NewEncoder(&b).Encode(m) + return b.Bytes(), err + default: + return nil, fmt.Errorf("format %q is not structured", format) + } +} + +// normalizeYAML converts yaml.v3's possible map[any]any values into the +// string-keyed maps used by the rest of the conversion pipeline. +func normalizeYAML(value any) any { + switch v := value.(type) { + case map[string]any: + out := make(map[string]any, len(v)) + for key, child := range v { + out[key] = normalizeYAML(child) + } + return out + case map[any]any: + out := make(map[string]any, len(v)) + for key, child := range v { + out[fmt.Sprint(key)] = normalizeYAML(child) + } + return out + case []any: + for i := range v { + v[i] = normalizeYAML(v[i]) + } + } + return value +} diff --git a/cmd/dataxl/table.go b/cmd/dataxl/table.go new file mode 100644 index 0000000..e4aef8a --- /dev/null +++ b/cmd/dataxl/table.go @@ -0,0 +1,265 @@ +package main + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "fmt" + "sort" + "strconv" + + "github.com/xuri/excelize/v2" +) + +// table is the common representation for CSV, TSV and XLSX. Rows are padded +// to Header width when read, which keeps subsequent conversions rectangular. +type table struct { + Header []string + Rows [][]string +} + +func parseTable(input []byte, format, sheet string) (table, error) { + switch format { + case "csv": + return readDelimited(input, ',') + case "tsv": + return readDelimited(input, '\t') + case "xlsx": + f, err := excelize.OpenReader(bytes.NewReader(input)) + if err != nil { + return table{}, err + } + defer f.Close() + rows, err := f.GetRows(sheet) + if err != nil { + return table{}, err + } + return rowsToTable(rows), nil + default: + return table{}, fmt.Errorf("format %q is not tabular", format) + } +} + +func readDelimited(input []byte, comma rune) (table, error) { + r := csv.NewReader(bytes.NewReader(input)) + r.Comma = comma + r.FieldsPerRecord = -1 + r.TrimLeadingSpace = comma == ',' + rows, err := r.ReadAll() + if err != nil { + return table{}, err + } + return rowsToTable(rows), nil +} + +func rowsToTable(rows [][]string) table { + if len(rows) == 0 { + return table{} + } + width := 0 + for _, row := range rows { + width = max(width, len(row)) + } + header := padRow(rows[0], width) + body := make([][]string, 0, len(rows)-1) + for _, row := range rows[1:] { + body = append(body, padRow(row, width)) + } + return table{Header: header, Rows: body} +} + +func padRow(row []string, width int) []string { + out := make([]string, width) + copy(out, row) + return out +} + +func encodeTable(t table, format, sheet string) ([]byte, error) { + switch format { + case "csv": + return writeDelimited(t, ',') + case "tsv": + return writeDelimited(t, '\t') + case "xlsx": + return writeXLSX(t, sheet) + default: + return nil, fmt.Errorf("format %q is not tabular", format) + } +} + +func writeDelimited(t table, comma rune) ([]byte, error) { + var b bytes.Buffer + w := csv.NewWriter(&b) + w.Comma = comma + if err := w.Write(t.Header); err != nil { + return nil, err + } + for _, row := range t.Rows { + if err := w.Write(row); err != nil { + return nil, err + } + } + w.Flush() + return b.Bytes(), w.Error() +} + +func writeXLSX(t table, sheet string) ([]byte, error) { + f := excelize.NewFile() + defer f.Close() + defaultSheet := f.GetSheetName(0) + if sheet == "" { + sheet = "Sheet1" + } + if defaultSheet != sheet { + if err := f.SetSheetName(defaultSheet, sheet); err != nil { + return nil, err + } + } + rows := append([][]string{t.Header}, t.Rows...) + for rowIndex, row := range rows { + for columnIndex, value := range row { + cell, err := excelize.CoordinatesToCellName(columnIndex+1, rowIndex+1) + if err != nil { + return nil, err + } + if err := f.SetCellValue(sheet, cell, value); err != nil { + return nil, err + } + } + } + if err := styleHeader(f, sheet, len(t.Header)); err != nil { + return nil, err + } + var b bytes.Buffer + if err := f.Write(&b); err != nil { + return nil, err + } + return b.Bytes(), nil +} + +// styleHeader applies editing conveniences without trying to preserve or +// emulate arbitrary workbook formatting. +func styleHeader(f *excelize.File, sheet string, width int) error { + if width == 0 { + return nil + } + end, err := excelize.CoordinatesToCellName(width, 1) + if err != nil { + return err + } + style, err := f.NewStyle(&excelize.Style{Font: &excelize.Font{Bold: true}}) + if err != nil { + return err + } + if err := f.SetCellStyle(sheet, "A1", end, style); err != nil { + return err + } + return f.SetPanes(sheet, &excelize.Panes{ + Freeze: true, YSplit: 1, TopLeftCell: "A2", ActivePane: "bottomLeft", + }) +} + +func valueToTable(value any) table { + records := recordsFromValue(value) + flatRows := make([]map[string]string, 0, len(records)) + seen := make(map[string]bool) + var header []string + for _, record := range records { + flat := make(map[string]string) + flatten("", record, flat) + for key := range flat { + if !seen[key] { + seen[key] = true + header = append(header, key) + } + } + flatRows = append(flatRows, flat) + } + // Stable column ordering makes generated files and their diffs predictable. + sort.Strings(header) + rows := make([][]string, 0, len(flatRows)) + for _, flat := range flatRows { + row := make([]string, len(header)) + for i, key := range header { + row[i] = flat[key] + } + rows = append(rows, row) + } + return table{Header: header, Rows: rows} +} + +func recordsFromValue(value any) []any { + switch v := value.(type) { + case []any: + return v + case map[string]any: + // Wrapper keys let object-shaped formats such as TOML carry row arrays. + for _, key := range []string{"rows", "records", "items"} { + if rows, ok := v[key].([]any); ok { + return rows + } + } + return []any{v} + default: + return []any{v} + } +} + +// flatten converts nested values to column paths such as user.name and +// items[0].sku. Empty maps and arrays use explicit textual markers; these are +// visible to editors but currently return as strings when converted back. +func flatten(prefix string, value any, out map[string]string) { + switch v := value.(type) { + case map[string]any: + if len(v) == 0 && prefix != "" { + out[prefix] = "{}" + return + } + keys := make([]string, 0, len(v)) + for key := range v { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + childPrefix := key + if prefix != "" { + childPrefix = prefix + "." + key + } + flatten(childPrefix, v[key], out) + } + case []any: + if len(v) == 0 && prefix != "" { + out[prefix] = "[]" + return + } + for i, child := range v { + flatten(fmt.Sprintf("%s[%d]", prefix, i), child, out) + } + default: + if prefix == "" { + prefix = "value" + } + out[prefix] = scalarString(v) + } +} + +func scalarString(value any) string { + switch v := value.(type) { + case nil: + return "" + case json.Number: + return v.String() + case string: + return v + case bool: + return strconv.FormatBool(v) + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64: + return fmt.Sprint(v) + default: + data, err := json.Marshal(v) + if err != nil { + return fmt.Sprint(v) + } + return string(data) + } +}