Refactor conversion pipeline by responsibility
This commit is contained in:
71
cmd/dataxl/conversion.go
Normal file
71
cmd/dataxl/conversion.go
Normal file
@@ -0,0 +1,71 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// convert routes data through either the structured-value or table internal
|
||||
// representation. Keeping this matrix in one place makes format adapters
|
||||
// independent from CLI and file I/O concerns.
|
||||
func convert(input []byte, from, to, sheet string, pretty bool) ([]byte, error) {
|
||||
switch {
|
||||
case isTabular(from) && isTabular(to):
|
||||
t, err := parseTable(input, from, sheet)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return encodeTable(t, to, sheet)
|
||||
case isTabular(from):
|
||||
t, err := parseTable(input, from, sheet)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return encodeStructured(tableToRecords(t), to, pretty)
|
||||
case isTabular(to):
|
||||
value, err := parseStructured(input, from)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return encodeTable(valueToTable(value), to, sheet)
|
||||
default:
|
||||
value, err := parseStructured(input, from)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return encodeStructured(value, to, pretty)
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeFormat(format string) string {
|
||||
format = strings.ToLower(strings.TrimPrefix(strings.TrimSpace(format), "."))
|
||||
switch format {
|
||||
case "yml":
|
||||
return "yaml"
|
||||
case "xlsm", "xls":
|
||||
return "xlsx"
|
||||
default:
|
||||
return format
|
||||
}
|
||||
}
|
||||
|
||||
func inferFormat(path string) string {
|
||||
if path == "" || path == "-" {
|
||||
return ""
|
||||
}
|
||||
return normalizeFormat(filepath.Ext(path))
|
||||
}
|
||||
|
||||
func validateFormat(format string) error {
|
||||
switch format {
|
||||
case "json", "yaml", "toml", "csv", "tsv", "xlsx":
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("unsupported format %q", format)
|
||||
}
|
||||
}
|
||||
|
||||
func isTabular(format string) bool {
|
||||
return format == "csv" || format == "tsv" || format == "xlsx"
|
||||
}
|
||||
7
cmd/dataxl/doc.go
Normal file
7
cmd/dataxl/doc.go
Normal file
@@ -0,0 +1,7 @@
|
||||
// Package main implements the dataxl command-line converter.
|
||||
//
|
||||
// Conversion uses two internal representations: structured Go values for
|
||||
// JSON/YAML/TOML and table for CSV/TSV/XLSX. Nested structured values cross the
|
||||
// boundary through dotted and indexed column paths such as user.name and
|
||||
// items[0].sku.
|
||||
package main
|
||||
@@ -1,25 +1,15 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
"github.com/xuri/excelize/v2"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// options contains CLI concerns only. Conversion functions receive explicit
|
||||
// arguments so they can be reused and tested without constructing a FlagSet.
|
||||
type options struct {
|
||||
inFile string
|
||||
outFile string
|
||||
@@ -29,11 +19,6 @@ type options struct {
|
||||
pretty bool
|
||||
}
|
||||
|
||||
type table struct {
|
||||
Header []string
|
||||
Rows [][]string
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(os.Args[1:], os.Stdin, os.Stdout, os.Stderr); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "dataxl:", err)
|
||||
@@ -42,6 +27,26 @@ func main() {
|
||||
}
|
||||
|
||||
func run(args []string, stdin io.Reader, stdout, stderr io.Writer) error {
|
||||
opt, err := parseOptions(args, stderr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := opt.resolveFormats(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
input, err := readInput(opt.inFile, stdin)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
output, err := convert(input, opt.from, opt.to, opt.sheet, opt.pretty)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeOutput(opt.outFile, stdout, output)
|
||||
}
|
||||
|
||||
func parseOptions(args []string, stderr io.Writer) (options, error) {
|
||||
var opt options
|
||||
fs := flag.NewFlagSet("dataxl", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
@@ -66,16 +71,15 @@ Notes:
|
||||
converting back to json/yaml/toml.`)
|
||||
}
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
return options{}, err
|
||||
}
|
||||
if fs.NArg() != 0 {
|
||||
return fmt.Errorf("unexpected arguments: %s", strings.Join(fs.Args(), " "))
|
||||
return options{}, fmt.Errorf("unexpected arguments: %s", strings.Join(fs.Args(), " "))
|
||||
}
|
||||
return opt, nil
|
||||
}
|
||||
|
||||
input, err := readInput(opt.inFile, stdin)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
func (opt *options) resolveFormats() error {
|
||||
opt.from = normalizeFormat(opt.from)
|
||||
opt.to = normalizeFormat(opt.to)
|
||||
if opt.from == "" {
|
||||
@@ -85,60 +89,12 @@ Notes:
|
||||
opt.to = inferFormat(opt.outFile)
|
||||
}
|
||||
if opt.from == "" || opt.to == "" {
|
||||
return errors.New("both -from and -to are required when a format cannot be inferred from file names")
|
||||
return fmt.Errorf("both -from and -to are required when a format cannot be inferred from file names")
|
||||
}
|
||||
if err := validateFormat(opt.from); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateFormat(opt.to); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var out []byte
|
||||
// Use table as the common representation whenever either side is a
|
||||
// spreadsheet-like format. Structured formats can then share the same
|
||||
// flatten/restore path for CSV, TSV, and XLSX.
|
||||
if isTabular(opt.from) && isTabular(opt.to) {
|
||||
t, err := parseTable(input, opt.from, opt.sheet)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out, err = encodeTable(t, opt.to, opt.sheet)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else if isTabular(opt.from) {
|
||||
t, err := parseTable(input, opt.from, opt.sheet)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records := tableToRecords(t)
|
||||
out, err = encodeStructured(records, opt.to, opt.pretty)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else if isTabular(opt.to) {
|
||||
value, err := parseStructured(input, opt.from)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
t := valueToTable(value)
|
||||
out, err = encodeTable(t, opt.to, opt.sheet)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
value, err := parseStructured(input, opt.from)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out, err = encodeStructured(value, opt.to, opt.pretty)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return writeOutput(opt.outFile, stdout, out)
|
||||
return validateFormat(opt.to)
|
||||
}
|
||||
|
||||
func readInput(path string, stdin io.Reader) ([]byte, error) {
|
||||
@@ -155,459 +111,3 @@ func writeOutput(path string, stdout io.Writer, data []byte) error {
|
||||
}
|
||||
return os.WriteFile(path, data, 0o644)
|
||||
}
|
||||
|
||||
func normalizeFormat(format string) string {
|
||||
format = strings.ToLower(strings.TrimPrefix(strings.TrimSpace(format), "."))
|
||||
switch format {
|
||||
case "yml":
|
||||
return "yaml"
|
||||
case "xlsm", "xls":
|
||||
return "xlsx"
|
||||
default:
|
||||
return format
|
||||
}
|
||||
}
|
||||
|
||||
func inferFormat(path string) string {
|
||||
if path == "" || path == "-" {
|
||||
return ""
|
||||
}
|
||||
return normalizeFormat(strings.TrimPrefix(filepath.Ext(path), "."))
|
||||
}
|
||||
|
||||
func validateFormat(format string) error {
|
||||
switch format {
|
||||
case "json", "yaml", "toml", "csv", "tsv", "xlsx":
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("unsupported format %q", format)
|
||||
}
|
||||
}
|
||||
|
||||
func isTabular(format string) bool {
|
||||
return format == "csv" || format == "tsv" || format == "xlsx"
|
||||
}
|
||||
|
||||
func parseStructured(input []byte, format string) (any, error) {
|
||||
var value any
|
||||
switch format {
|
||||
case "json":
|
||||
dec := json.NewDecoder(bytes.NewReader(input))
|
||||
dec.UseNumber()
|
||||
if err := dec.Decode(&value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case "yaml":
|
||||
if err := yaml.Unmarshal(input, &value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value = normalizeYAML(value)
|
||||
case "toml":
|
||||
var m map[string]any
|
||||
if err := toml.Unmarshal(input, &m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value = m
|
||||
default:
|
||||
return nil, fmt.Errorf("format %q is not structured", format)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func encodeStructured(value any, format string, pretty bool) ([]byte, error) {
|
||||
switch format {
|
||||
case "json":
|
||||
if pretty {
|
||||
return json.MarshalIndent(value, "", " ")
|
||||
}
|
||||
return json.Marshal(value)
|
||||
case "yaml":
|
||||
return yaml.Marshal(value)
|
||||
case "toml":
|
||||
m, ok := value.(map[string]any)
|
||||
if !ok {
|
||||
m = map[string]any{"rows": value}
|
||||
}
|
||||
var b bytes.Buffer
|
||||
err := toml.NewEncoder(&b).Encode(m)
|
||||
return b.Bytes(), err
|
||||
default:
|
||||
return nil, fmt.Errorf("format %q is not structured", format)
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeYAML(value any) any {
|
||||
switch v := value.(type) {
|
||||
case map[string]any:
|
||||
out := make(map[string]any, len(v))
|
||||
for key, child := range v {
|
||||
out[key] = normalizeYAML(child)
|
||||
}
|
||||
return out
|
||||
case map[any]any:
|
||||
out := make(map[string]any, len(v))
|
||||
for key, child := range v {
|
||||
out[fmt.Sprint(key)] = normalizeYAML(child)
|
||||
}
|
||||
return out
|
||||
case []any:
|
||||
for i := range v {
|
||||
v[i] = normalizeYAML(v[i])
|
||||
}
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func parseTable(input []byte, format, sheet string) (table, error) {
|
||||
switch format {
|
||||
case "csv":
|
||||
return readDelimited(input, ',')
|
||||
case "tsv":
|
||||
return readDelimited(input, '\t')
|
||||
case "xlsx":
|
||||
f, err := excelize.OpenReader(bytes.NewReader(input))
|
||||
if err != nil {
|
||||
return table{}, err
|
||||
}
|
||||
defer f.Close()
|
||||
rows, err := f.GetRows(sheet)
|
||||
if err != nil {
|
||||
return table{}, err
|
||||
}
|
||||
return rowsToTable(rows), nil
|
||||
default:
|
||||
return table{}, fmt.Errorf("format %q is not tabular", format)
|
||||
}
|
||||
}
|
||||
|
||||
func readDelimited(input []byte, comma rune) (table, error) {
|
||||
r := csv.NewReader(bytes.NewReader(input))
|
||||
r.Comma = comma
|
||||
r.FieldsPerRecord = -1
|
||||
r.TrimLeadingSpace = comma == ','
|
||||
rows, err := r.ReadAll()
|
||||
if err != nil {
|
||||
return table{}, err
|
||||
}
|
||||
return rowsToTable(rows), nil
|
||||
}
|
||||
|
||||
func rowsToTable(rows [][]string) table {
|
||||
if len(rows) == 0 {
|
||||
return table{}
|
||||
}
|
||||
width := 0
|
||||
for _, row := range rows {
|
||||
if len(row) > width {
|
||||
width = len(row)
|
||||
}
|
||||
}
|
||||
header := padRow(rows[0], width)
|
||||
body := make([][]string, 0, len(rows)-1)
|
||||
for _, row := range rows[1:] {
|
||||
body = append(body, padRow(row, width))
|
||||
}
|
||||
return table{Header: header, Rows: body}
|
||||
}
|
||||
|
||||
func padRow(row []string, width int) []string {
|
||||
out := make([]string, width)
|
||||
copy(out, row)
|
||||
return out
|
||||
}
|
||||
|
||||
func encodeTable(t table, format, sheet string) ([]byte, error) {
|
||||
switch format {
|
||||
case "csv":
|
||||
return writeDelimited(t, ',')
|
||||
case "tsv":
|
||||
return writeDelimited(t, '\t')
|
||||
case "xlsx":
|
||||
return writeXLSX(t, sheet)
|
||||
default:
|
||||
return nil, fmt.Errorf("format %q is not tabular", format)
|
||||
}
|
||||
}
|
||||
|
||||
func writeDelimited(t table, comma rune) ([]byte, error) {
|
||||
var b bytes.Buffer
|
||||
w := csv.NewWriter(&b)
|
||||
w.Comma = comma
|
||||
if err := w.Write(t.Header); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, row := range t.Rows {
|
||||
if err := w.Write(row); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
w.Flush()
|
||||
return b.Bytes(), w.Error()
|
||||
}
|
||||
|
||||
func writeXLSX(t table, sheet string) ([]byte, error) {
|
||||
f := excelize.NewFile()
|
||||
defaultSheet := f.GetSheetName(0)
|
||||
if sheet == "" {
|
||||
sheet = "Sheet1"
|
||||
}
|
||||
if defaultSheet != sheet {
|
||||
if err := f.SetSheetName(defaultSheet, sheet); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
rows := append([][]string{t.Header}, t.Rows...)
|
||||
for r, row := range rows {
|
||||
for c, value := range row {
|
||||
cell, err := excelize.CoordinatesToCellName(c+1, r+1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := f.SetCellValue(sheet, cell, value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(t.Header) > 0 {
|
||||
// The generated workbook is meant for editing, so keep the header row
|
||||
// visible and visually distinct.
|
||||
end, _ := excelize.CoordinatesToCellName(len(t.Header), 1)
|
||||
style, _ := f.NewStyle(&excelize.Style{Font: &excelize.Font{Bold: true}})
|
||||
_ = f.SetCellStyle(sheet, "A1", end, style)
|
||||
_ = f.SetPanes(sheet, &excelize.Panes{
|
||||
Freeze: true,
|
||||
Split: false,
|
||||
XSplit: 0,
|
||||
YSplit: 1,
|
||||
TopLeftCell: "A2",
|
||||
ActivePane: "bottomLeft",
|
||||
})
|
||||
}
|
||||
var b bytes.Buffer
|
||||
if err := f.Write(&b); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b.Bytes(), nil
|
||||
}
|
||||
|
||||
func valueToTable(value any) table {
|
||||
records := recordsFromValue(value)
|
||||
flatRows := make([]map[string]string, 0, len(records))
|
||||
seen := map[string]bool{}
|
||||
var header []string
|
||||
for _, record := range records {
|
||||
flat := map[string]string{}
|
||||
flatten("", record, flat)
|
||||
for key := range flat {
|
||||
if !seen[key] {
|
||||
seen[key] = true
|
||||
header = append(header, key)
|
||||
}
|
||||
}
|
||||
flatRows = append(flatRows, flat)
|
||||
}
|
||||
// Stable column ordering keeps generated CSV/TSV/XLSX diffs predictable.
|
||||
sort.Strings(header)
|
||||
rows := make([][]string, 0, len(flatRows))
|
||||
for _, flat := range flatRows {
|
||||
row := make([]string, len(header))
|
||||
for i, key := range header {
|
||||
row[i] = flat[key]
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
return table{Header: header, Rows: rows}
|
||||
}
|
||||
|
||||
func recordsFromValue(value any) []any {
|
||||
switch v := value.(type) {
|
||||
case []any:
|
||||
return v
|
||||
case map[string]any:
|
||||
// Common wrapper keys let TOML and object-shaped inputs represent a
|
||||
// table without adding a format-specific flag.
|
||||
for _, key := range []string{"rows", "records", "items"} {
|
||||
if rows, ok := v[key].([]any); ok {
|
||||
return rows
|
||||
}
|
||||
}
|
||||
return []any{v}
|
||||
default:
|
||||
return []any{v}
|
||||
}
|
||||
}
|
||||
|
||||
// flatten converts nested values into spreadsheet-safe column paths such as
|
||||
// user.name and items[0].sku.
|
||||
func flatten(prefix string, value any, out map[string]string) {
|
||||
switch v := value.(type) {
|
||||
case map[string]any:
|
||||
if len(v) == 0 && prefix != "" {
|
||||
out[prefix] = "{}"
|
||||
return
|
||||
}
|
||||
keys := make([]string, 0, len(v))
|
||||
for key := range v {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
childPrefix := key
|
||||
if prefix != "" {
|
||||
childPrefix = prefix + "." + key
|
||||
}
|
||||
flatten(childPrefix, v[key], out)
|
||||
}
|
||||
case []any:
|
||||
if len(v) == 0 && prefix != "" {
|
||||
out[prefix] = "[]"
|
||||
return
|
||||
}
|
||||
for i, child := range v {
|
||||
flatten(fmt.Sprintf("%s[%d]", prefix, i), child, out)
|
||||
}
|
||||
default:
|
||||
if prefix == "" {
|
||||
prefix = "value"
|
||||
}
|
||||
out[prefix] = scalarString(v)
|
||||
}
|
||||
}
|
||||
|
||||
func scalarString(value any) string {
|
||||
switch v := value.(type) {
|
||||
case nil:
|
||||
return ""
|
||||
case json.Number:
|
||||
return v.String()
|
||||
case string:
|
||||
return v
|
||||
case bool:
|
||||
return strconv.FormatBool(v)
|
||||
case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64:
|
||||
return fmt.Sprint(v)
|
||||
default:
|
||||
data, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return fmt.Sprint(v)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
}
|
||||
|
||||
// tableToRecords restores each row by interpreting header cells as path
|
||||
// expressions. Empty headers are ignored so spare spreadsheet columns are safe.
|
||||
func tableToRecords(t table) []map[string]any {
|
||||
var records []map[string]any
|
||||
for _, row := range t.Rows {
|
||||
record := map[string]any{}
|
||||
for i, header := range t.Header {
|
||||
header = strings.TrimSpace(header)
|
||||
if header == "" || i >= len(row) {
|
||||
continue
|
||||
}
|
||||
setPath(record, header, parseCell(row[i]))
|
||||
}
|
||||
records = append(records, record)
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
// parseCell keeps spreadsheet round trips useful while avoiding broad type
|
||||
// inference that could surprise users editing IDs or codes.
|
||||
func parseCell(s string) any {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
if s == "true" {
|
||||
return true
|
||||
}
|
||||
if s == "false" {
|
||||
return false
|
||||
}
|
||||
if i, err := strconv.ParseInt(s, 10, 64); err == nil {
|
||||
return i
|
||||
}
|
||||
if f, err := strconv.ParseFloat(s, 64); err == nil && strings.ContainsAny(s, ".eE") {
|
||||
return f
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
var pathTokenRE = regexp.MustCompile(`([^\.\[\]]+)|\[(\d+)\]`)
|
||||
|
||||
// setPath creates maps and slices as needed for dotted and indexed header
|
||||
// paths. Invalid intermediate shapes are left unchanged instead of guessing.
|
||||
func setPath(root map[string]any, path string, value any) {
|
||||
tokens := parsePath(path)
|
||||
if len(tokens) == 0 {
|
||||
return
|
||||
}
|
||||
var cur any = root
|
||||
for i, token := range tokens {
|
||||
last := i == len(tokens)-1
|
||||
nextIsIndex := !last && tokens[i+1].isIndex
|
||||
if token.isIndex {
|
||||
continue
|
||||
}
|
||||
m, ok := cur.(map[string]any)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if last {
|
||||
m[token.key] = value
|
||||
return
|
||||
}
|
||||
if _, ok := m[token.key]; !ok {
|
||||
if nextIsIndex {
|
||||
m[token.key] = []any{}
|
||||
} else {
|
||||
m[token.key] = map[string]any{}
|
||||
}
|
||||
}
|
||||
if nextIsIndex {
|
||||
slice, _ := m[token.key].([]any)
|
||||
index := tokens[i+1].index
|
||||
for len(slice) <= index {
|
||||
if i+2 < len(tokens) && !tokens[i+2].isIndex {
|
||||
slice = append(slice, map[string]any{})
|
||||
} else {
|
||||
slice = append(slice, nil)
|
||||
}
|
||||
}
|
||||
m[token.key] = slice
|
||||
if i+2 == len(tokens) {
|
||||
slice[index] = value
|
||||
return
|
||||
}
|
||||
if slice[index] == nil {
|
||||
slice[index] = map[string]any{}
|
||||
}
|
||||
cur = slice[index]
|
||||
i++
|
||||
continue
|
||||
}
|
||||
cur = m[token.key]
|
||||
}
|
||||
}
|
||||
|
||||
type pathToken struct {
|
||||
key string
|
||||
index int
|
||||
isIndex bool
|
||||
}
|
||||
|
||||
func parsePath(path string) []pathToken {
|
||||
matches := pathTokenRE.FindAllStringSubmatch(path, -1)
|
||||
tokens := make([]pathToken, 0, len(matches))
|
||||
for _, m := range matches {
|
||||
if m[1] != "" {
|
||||
tokens = append(tokens, pathToken{key: m[1]})
|
||||
continue
|
||||
}
|
||||
index, _ := strconv.Atoi(m[2])
|
||||
tokens = append(tokens, pathToken{index: index, isIndex: true})
|
||||
}
|
||||
return tokens
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -88,6 +89,76 @@ func TestJSONToXLSXAndBack(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveFormatsFromFileExtensions(t *testing.T) {
|
||||
opt := options{inFile: "input.YML", outFile: "output.XLSM"}
|
||||
if err := opt.resolveFormats(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if opt.from != "yaml" || opt.to != "xlsx" {
|
||||
t.Fatalf("resolved formats = %q -> %q, want yaml -> xlsx", opt.from, opt.to)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertStructuredToStructured(t *testing.T) {
|
||||
got, err := convert([]byte("name: Alice\nactive: true\n"), "yaml", "json", "Sheet1", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != `{"active":true,"name":"Alice"}` {
|
||||
t.Fatalf("JSON = %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRowsToTablePadsRaggedRows(t *testing.T) {
|
||||
got := rowsToTable([][]string{{"a", "b"}, {"1"}, {"2", "3", "4"}})
|
||||
want := table{
|
||||
Header: []string{"a", "b", ""},
|
||||
Rows: [][]string{{"1", "", ""}, {"2", "3", "4"}},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("table = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCellConservativeInference(t *testing.T) {
|
||||
tests := map[string]any{
|
||||
"": "",
|
||||
"true": true,
|
||||
"42": int64(42),
|
||||
"3.14": 3.14,
|
||||
"00123": "00123",
|
||||
"1e3": 1000.0,
|
||||
"Alice": "Alice",
|
||||
}
|
||||
for input, want := range tests {
|
||||
if got := parseCell(input); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("parseCell(%q) = %#v, want %#v", input, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetPathDoesNotOverwriteConflictingShape(t *testing.T) {
|
||||
record := map[string]any{}
|
||||
setPath(record, "user", "Alice")
|
||||
setPath(record, "user.name", "Bob")
|
||||
if got := record["user"]; got != "Alice" {
|
||||
t.Fatalf("user = %#v, want original scalar", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetPathRestoresNestedArrays(t *testing.T) {
|
||||
record := map[string]any{}
|
||||
setPath(record, "orders[0].items[1].sku", "B-002")
|
||||
|
||||
orders := record["orders"].([]any)
|
||||
order := orders[0].(map[string]any)
|
||||
items := order["items"].([]any)
|
||||
item := items[1].(map[string]any)
|
||||
if item["sku"] != "B-002" {
|
||||
t.Fatalf("nested sku = %#v, want B-002", item["sku"])
|
||||
}
|
||||
}
|
||||
|
||||
type ioDiscard struct{}
|
||||
|
||||
func (ioDiscard) Write(p []byte) (int, error) { return len(p), nil }
|
||||
|
||||
134
cmd/dataxl/path.go
Normal file
134
cmd/dataxl/path.go
Normal file
@@ -0,0 +1,134 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// tableToRecords interprets non-empty headers as path expressions. Blank
|
||||
// headers are intentionally ignored so spare spreadsheet columns are harmless.
|
||||
func tableToRecords(t table) []map[string]any {
|
||||
records := make([]map[string]any, 0, len(t.Rows))
|
||||
for _, row := range t.Rows {
|
||||
record := make(map[string]any)
|
||||
for i, header := range t.Header {
|
||||
header = strings.TrimSpace(header)
|
||||
if header == "" || i >= len(row) {
|
||||
continue
|
||||
}
|
||||
setPath(record, header, parseCell(row[i]))
|
||||
}
|
||||
records = append(records, record)
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
// parseCell deliberately performs narrow inference. In particular, strings
|
||||
// such as 00123 remain strings because converting identifiers is surprising.
|
||||
func parseCell(s string) any {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
if s == "true" {
|
||||
return true
|
||||
}
|
||||
if s == "false" {
|
||||
return false
|
||||
}
|
||||
// Preserve zero-padded identifiers such as postal codes and product codes.
|
||||
leadingZero := len(s) > 1 && s[0] == '0'
|
||||
negativeLeadingZero := len(s) > 2 && s[0] == '-' && s[1] == '0'
|
||||
if i, err := strconv.ParseInt(s, 10, 64); err == nil && !leadingZero && !negativeLeadingZero {
|
||||
return i
|
||||
}
|
||||
if f, err := strconv.ParseFloat(s, 64); err == nil && strings.ContainsAny(s, ".eE") {
|
||||
return f
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
var pathTokenRE = regexp.MustCompile(`([^\.\[\]]+)|\[(\d+)\]`)
|
||||
|
||||
type pathToken struct {
|
||||
key string
|
||||
index int
|
||||
isIndex bool
|
||||
}
|
||||
|
||||
func parsePath(path string) []pathToken {
|
||||
matches := pathTokenRE.FindAllStringSubmatch(path, -1)
|
||||
tokens := make([]pathToken, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
if match[1] != "" {
|
||||
tokens = append(tokens, pathToken{key: match[1]})
|
||||
continue
|
||||
}
|
||||
index, _ := strconv.Atoi(match[2]) // regexp guarantees decimal digits.
|
||||
tokens = append(tokens, pathToken{index: index, isIndex: true})
|
||||
}
|
||||
return tokens
|
||||
}
|
||||
|
||||
// setPath creates maps and slices while walking a dotted/indexed header. The
|
||||
// supported grammar is a sequence of map keys with optional array indices,
|
||||
// e.g. "orders[0].items[1].sku". Conflicting intermediate shapes are left
|
||||
// unchanged rather than silently overwriting data from an earlier column.
|
||||
func setPath(root map[string]any, path string, value any) {
|
||||
tokens := parsePath(path)
|
||||
if len(tokens) == 0 {
|
||||
return
|
||||
}
|
||||
var current any = root
|
||||
for i := 0; i < len(tokens); i++ {
|
||||
token := tokens[i]
|
||||
last := i == len(tokens)-1
|
||||
nextIsIndex := !last && tokens[i+1].isIndex
|
||||
if token.isIndex {
|
||||
continue
|
||||
}
|
||||
m, ok := current.(map[string]any)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if last {
|
||||
m[token.key] = value
|
||||
return
|
||||
}
|
||||
if _, exists := m[token.key]; !exists {
|
||||
if nextIsIndex {
|
||||
m[token.key] = []any{}
|
||||
} else {
|
||||
m[token.key] = map[string]any{}
|
||||
}
|
||||
}
|
||||
if !nextIsIndex {
|
||||
current = m[token.key]
|
||||
continue
|
||||
}
|
||||
|
||||
slice, ok := m[token.key].([]any)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
index := tokens[i+1].index
|
||||
for len(slice) <= index {
|
||||
if i+2 < len(tokens) && !tokens[i+2].isIndex {
|
||||
slice = append(slice, map[string]any{})
|
||||
} else {
|
||||
slice = append(slice, nil)
|
||||
}
|
||||
}
|
||||
m[token.key] = slice
|
||||
if i+2 == len(tokens) {
|
||||
slice[index] = value
|
||||
return
|
||||
}
|
||||
if slice[index] == nil {
|
||||
slice[index] = map[string]any{}
|
||||
}
|
||||
current = slice[index]
|
||||
i++ // The array index was consumed together with its key.
|
||||
}
|
||||
}
|
||||
85
cmd/dataxl/structured.go
Normal file
85
cmd/dataxl/structured.go
Normal file
@@ -0,0 +1,85 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// parseStructured decodes a format into maps, slices and scalar Go values.
|
||||
// json.Number is retained so large JSON integers do not pass through float64.
|
||||
func parseStructured(input []byte, format string) (any, error) {
|
||||
var value any
|
||||
switch format {
|
||||
case "json":
|
||||
dec := json.NewDecoder(bytes.NewReader(input))
|
||||
dec.UseNumber()
|
||||
if err := dec.Decode(&value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case "yaml":
|
||||
if err := yaml.Unmarshal(input, &value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value = normalizeYAML(value)
|
||||
case "toml":
|
||||
var m map[string]any
|
||||
if err := toml.Unmarshal(input, &m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value = m
|
||||
default:
|
||||
return nil, fmt.Errorf("format %q is not structured", format)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func encodeStructured(value any, format string, pretty bool) ([]byte, error) {
|
||||
switch format {
|
||||
case "json":
|
||||
if pretty {
|
||||
return json.MarshalIndent(value, "", " ")
|
||||
}
|
||||
return json.Marshal(value)
|
||||
case "yaml":
|
||||
return yaml.Marshal(value)
|
||||
case "toml":
|
||||
m, ok := value.(map[string]any)
|
||||
if !ok {
|
||||
// TOML has no top-level array, so preserve it under a documented key.
|
||||
m = map[string]any{"rows": value}
|
||||
}
|
||||
var b bytes.Buffer
|
||||
err := toml.NewEncoder(&b).Encode(m)
|
||||
return b.Bytes(), err
|
||||
default:
|
||||
return nil, fmt.Errorf("format %q is not structured", format)
|
||||
}
|
||||
}
|
||||
|
||||
// normalizeYAML converts yaml.v3's possible map[any]any values into the
|
||||
// string-keyed maps used by the rest of the conversion pipeline.
|
||||
func normalizeYAML(value any) any {
|
||||
switch v := value.(type) {
|
||||
case map[string]any:
|
||||
out := make(map[string]any, len(v))
|
||||
for key, child := range v {
|
||||
out[key] = normalizeYAML(child)
|
||||
}
|
||||
return out
|
||||
case map[any]any:
|
||||
out := make(map[string]any, len(v))
|
||||
for key, child := range v {
|
||||
out[fmt.Sprint(key)] = normalizeYAML(child)
|
||||
}
|
||||
return out
|
||||
case []any:
|
||||
for i := range v {
|
||||
v[i] = normalizeYAML(v[i])
|
||||
}
|
||||
}
|
||||
return value
|
||||
}
|
||||
265
cmd/dataxl/table.go
Normal file
265
cmd/dataxl/table.go
Normal file
@@ -0,0 +1,265 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
|
||||
"github.com/xuri/excelize/v2"
|
||||
)
|
||||
|
||||
// table is the common representation for CSV, TSV and XLSX. Rows are padded
|
||||
// to Header width when read, which keeps subsequent conversions rectangular.
|
||||
type table struct {
|
||||
Header []string
|
||||
Rows [][]string
|
||||
}
|
||||
|
||||
func parseTable(input []byte, format, sheet string) (table, error) {
|
||||
switch format {
|
||||
case "csv":
|
||||
return readDelimited(input, ',')
|
||||
case "tsv":
|
||||
return readDelimited(input, '\t')
|
||||
case "xlsx":
|
||||
f, err := excelize.OpenReader(bytes.NewReader(input))
|
||||
if err != nil {
|
||||
return table{}, err
|
||||
}
|
||||
defer f.Close()
|
||||
rows, err := f.GetRows(sheet)
|
||||
if err != nil {
|
||||
return table{}, err
|
||||
}
|
||||
return rowsToTable(rows), nil
|
||||
default:
|
||||
return table{}, fmt.Errorf("format %q is not tabular", format)
|
||||
}
|
||||
}
|
||||
|
||||
func readDelimited(input []byte, comma rune) (table, error) {
|
||||
r := csv.NewReader(bytes.NewReader(input))
|
||||
r.Comma = comma
|
||||
r.FieldsPerRecord = -1
|
||||
r.TrimLeadingSpace = comma == ','
|
||||
rows, err := r.ReadAll()
|
||||
if err != nil {
|
||||
return table{}, err
|
||||
}
|
||||
return rowsToTable(rows), nil
|
||||
}
|
||||
|
||||
func rowsToTable(rows [][]string) table {
|
||||
if len(rows) == 0 {
|
||||
return table{}
|
||||
}
|
||||
width := 0
|
||||
for _, row := range rows {
|
||||
width = max(width, len(row))
|
||||
}
|
||||
header := padRow(rows[0], width)
|
||||
body := make([][]string, 0, len(rows)-1)
|
||||
for _, row := range rows[1:] {
|
||||
body = append(body, padRow(row, width))
|
||||
}
|
||||
return table{Header: header, Rows: body}
|
||||
}
|
||||
|
||||
func padRow(row []string, width int) []string {
|
||||
out := make([]string, width)
|
||||
copy(out, row)
|
||||
return out
|
||||
}
|
||||
|
||||
func encodeTable(t table, format, sheet string) ([]byte, error) {
|
||||
switch format {
|
||||
case "csv":
|
||||
return writeDelimited(t, ',')
|
||||
case "tsv":
|
||||
return writeDelimited(t, '\t')
|
||||
case "xlsx":
|
||||
return writeXLSX(t, sheet)
|
||||
default:
|
||||
return nil, fmt.Errorf("format %q is not tabular", format)
|
||||
}
|
||||
}
|
||||
|
||||
func writeDelimited(t table, comma rune) ([]byte, error) {
|
||||
var b bytes.Buffer
|
||||
w := csv.NewWriter(&b)
|
||||
w.Comma = comma
|
||||
if err := w.Write(t.Header); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, row := range t.Rows {
|
||||
if err := w.Write(row); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
w.Flush()
|
||||
return b.Bytes(), w.Error()
|
||||
}
|
||||
|
||||
func writeXLSX(t table, sheet string) ([]byte, error) {
|
||||
f := excelize.NewFile()
|
||||
defer f.Close()
|
||||
defaultSheet := f.GetSheetName(0)
|
||||
if sheet == "" {
|
||||
sheet = "Sheet1"
|
||||
}
|
||||
if defaultSheet != sheet {
|
||||
if err := f.SetSheetName(defaultSheet, sheet); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
rows := append([][]string{t.Header}, t.Rows...)
|
||||
for rowIndex, row := range rows {
|
||||
for columnIndex, value := range row {
|
||||
cell, err := excelize.CoordinatesToCellName(columnIndex+1, rowIndex+1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := f.SetCellValue(sheet, cell, value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := styleHeader(f, sheet, len(t.Header)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var b bytes.Buffer
|
||||
if err := f.Write(&b); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b.Bytes(), nil
|
||||
}
|
||||
|
||||
// styleHeader applies editing conveniences without trying to preserve or
|
||||
// emulate arbitrary workbook formatting.
|
||||
func styleHeader(f *excelize.File, sheet string, width int) error {
|
||||
if width == 0 {
|
||||
return nil
|
||||
}
|
||||
end, err := excelize.CoordinatesToCellName(width, 1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
style, err := f.NewStyle(&excelize.Style{Font: &excelize.Font{Bold: true}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := f.SetCellStyle(sheet, "A1", end, style); err != nil {
|
||||
return err
|
||||
}
|
||||
return f.SetPanes(sheet, &excelize.Panes{
|
||||
Freeze: true, YSplit: 1, TopLeftCell: "A2", ActivePane: "bottomLeft",
|
||||
})
|
||||
}
|
||||
|
||||
func valueToTable(value any) table {
|
||||
records := recordsFromValue(value)
|
||||
flatRows := make([]map[string]string, 0, len(records))
|
||||
seen := make(map[string]bool)
|
||||
var header []string
|
||||
for _, record := range records {
|
||||
flat := make(map[string]string)
|
||||
flatten("", record, flat)
|
||||
for key := range flat {
|
||||
if !seen[key] {
|
||||
seen[key] = true
|
||||
header = append(header, key)
|
||||
}
|
||||
}
|
||||
flatRows = append(flatRows, flat)
|
||||
}
|
||||
// Stable column ordering makes generated files and their diffs predictable.
|
||||
sort.Strings(header)
|
||||
rows := make([][]string, 0, len(flatRows))
|
||||
for _, flat := range flatRows {
|
||||
row := make([]string, len(header))
|
||||
for i, key := range header {
|
||||
row[i] = flat[key]
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
return table{Header: header, Rows: rows}
|
||||
}
|
||||
|
||||
func recordsFromValue(value any) []any {
|
||||
switch v := value.(type) {
|
||||
case []any:
|
||||
return v
|
||||
case map[string]any:
|
||||
// Wrapper keys let object-shaped formats such as TOML carry row arrays.
|
||||
for _, key := range []string{"rows", "records", "items"} {
|
||||
if rows, ok := v[key].([]any); ok {
|
||||
return rows
|
||||
}
|
||||
}
|
||||
return []any{v}
|
||||
default:
|
||||
return []any{v}
|
||||
}
|
||||
}
|
||||
|
||||
// flatten converts nested values to column paths such as user.name and
|
||||
// items[0].sku. Empty maps and arrays use explicit textual markers; these are
|
||||
// visible to editors but currently return as strings when converted back.
|
||||
func flatten(prefix string, value any, out map[string]string) {
|
||||
switch v := value.(type) {
|
||||
case map[string]any:
|
||||
if len(v) == 0 && prefix != "" {
|
||||
out[prefix] = "{}"
|
||||
return
|
||||
}
|
||||
keys := make([]string, 0, len(v))
|
||||
for key := range v {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
childPrefix := key
|
||||
if prefix != "" {
|
||||
childPrefix = prefix + "." + key
|
||||
}
|
||||
flatten(childPrefix, v[key], out)
|
||||
}
|
||||
case []any:
|
||||
if len(v) == 0 && prefix != "" {
|
||||
out[prefix] = "[]"
|
||||
return
|
||||
}
|
||||
for i, child := range v {
|
||||
flatten(fmt.Sprintf("%s[%d]", prefix, i), child, out)
|
||||
}
|
||||
default:
|
||||
if prefix == "" {
|
||||
prefix = "value"
|
||||
}
|
||||
out[prefix] = scalarString(v)
|
||||
}
|
||||
}
|
||||
|
||||
func scalarString(value any) string {
|
||||
switch v := value.(type) {
|
||||
case nil:
|
||||
return ""
|
||||
case json.Number:
|
||||
return v.String()
|
||||
case string:
|
||||
return v
|
||||
case bool:
|
||||
return strconv.FormatBool(v)
|
||||
case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64:
|
||||
return fmt.Sprint(v)
|
||||
default:
|
||||
data, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return fmt.Sprint(v)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user