feat: add keyed record merge (#9)
CI / test (pull_request) Failing after 39s

This commit is contained in:
2026-10-26 15:55:49 +09:00
parent 4c524e7a24
commit d3a08c25e2
6 changed files with 489 additions and 9 deletions
+57 -9
View File
@@ -14,13 +14,16 @@ var buildVersion = "dev"
// options contains CLI concerns only. Conversion functions receive explicit
// arguments so they can be reused and tested without constructing a FlagSet.
type options struct {
inFile string
outFile string
from string
to string
sheet string
pretty bool
version bool
inFile string
outFile string
from string
to string
sheet string
pretty bool
version bool
mergeFile string
mergeFrom string
mergeOn string
}
func main() {
@@ -47,7 +50,22 @@ func run(args []string, stdin io.Reader, stdout, stderr io.Writer) error {
if err != nil {
return err
}
output, err := convert(input, opt.from, opt.to, opt.sheet, opt.pretty)
var output []byte
if opt.mergeFile != "" {
if isStdinPath(opt.inFile) && isStdinPath(opt.mergeFile) {
return fmt.Errorf("primary and merge inputs cannot both use stdin")
}
mergeInput, readErr := readInput(opt.mergeFile, stdin)
if readErr != nil {
return readErr
}
output, err = mergeAndConvert(
input, opt.from, mergeInput, opt.mergeFrom, opt.mergeOn,
opt.to, opt.sheet, opt.pretty,
)
} else {
output, err = convert(input, opt.from, opt.to, opt.sheet, opt.pretty)
}
if err != nil {
return err
}
@@ -65,11 +83,15 @@ func parseOptions(args []string, stderr io.Writer) (options, error) {
fs.StringVar(&opt.sheet, "sheet", "Sheet1", "worksheet name for xlsx input/output")
fs.BoolVar(&opt.pretty, "pretty", true, "pretty-print structured output")
fs.BoolVar(&opt.version, "version", false, "print version and exit")
fs.StringVar(&opt.mergeFile, "merge", "", "second input file used to enrich primary records")
fs.StringVar(&opt.mergeFrom, "merge-from", "", "second input format; inferred from -merge when omitted")
fs.StringVar(&opt.mergeOn, "on", "", "column path used to match records when -merge is set")
fs.Usage = func() {
_, _ = fmt.Fprintln(stderr, `Usage:
dataxl -from yaml -to tsv -i input.yaml -o output.tsv
dataxl -from tsv -to json < paste-from-excel.tsv
dataxl -from json -to xlsx -i data.json -o data.xlsx
dataxl -i people.csv -merge details.json -on id -to yaml
Formats:
json, yaml/yml, toml, csv, tsv, xlsx, markdown/md
@@ -102,6 +124,7 @@ func resolvedVersion() string {
func (opt *options) resolveFormats() error {
opt.from = normalizeFormat(opt.from)
opt.to = normalizeFormat(opt.to)
opt.mergeFrom = normalizeFormat(opt.mergeFrom)
if opt.from == "" {
opt.from = inferFormat(opt.inFile)
}
@@ -114,7 +137,32 @@ func (opt *options) resolveFormats() error {
if err := validateFormat(opt.from); err != nil {
return err
}
return validateFormat(opt.to)
if err := validateFormat(opt.to); err != nil {
return err
}
if opt.mergeFile == "" {
if opt.mergeFrom != "" || opt.mergeOn != "" {
return fmt.Errorf("-merge-from and -on require -merge")
}
return nil
}
if opt.mergeFrom == "" {
opt.mergeFrom = inferFormat(opt.mergeFile)
}
if opt.mergeFrom == "" {
return fmt.Errorf("-merge-from is required when the second input format cannot be inferred")
}
if err := validateFormat(opt.mergeFrom); err != nil {
return fmt.Errorf("merge input: %w", err)
}
if strings.TrimSpace(opt.mergeOn) == "" {
return fmt.Errorf("-on is required when -merge is set")
}
return nil
}
func isStdinPath(path string) bool {
return path == "" || path == "-"
}
func readInput(path string, stdin io.Reader) ([]byte, error) {
+255
View File
@@ -0,0 +1,255 @@
package main
import (
"encoding/json"
"fmt"
"math"
"math/big"
"reflect"
"strconv"
)
// mergeAndConvert parses two record sets, enriches primary records with fields
// from matching secondary records, and encodes the result in the requested
// output format. Primary order and unmatched primary records are preserved.
func mergeAndConvert(
primary []byte,
primaryFormat string,
secondary []byte,
secondaryFormat string,
on string,
to string,
sheet string,
pretty bool,
) ([]byte, error) {
primaryRecords, err := parseMergeRecords(primary, primaryFormat, sheet)
if err != nil {
return nil, fmt.Errorf("primary input: %w", err)
}
secondaryRecords, err := parseMergeRecords(secondary, secondaryFormat, sheet)
if err != nil {
return nil, fmt.Errorf("merge input: %w", err)
}
merged, err := mergeRecords(primaryRecords, secondaryRecords, on)
if err != nil {
return nil, err
}
return encodeMergedRecords(merged, to, sheet, pretty)
}
func parseMergeRecords(input []byte, format, sheet string) ([]map[string]any, error) {
if isTabular(format) {
t, err := parseTable(input, format, sheet)
if err != nil {
return nil, err
}
return tableToRecords(t)
}
value, err := parseStructured(input, format)
if err != nil {
return nil, err
}
values := recordsFromValue(value)
records := make([]map[string]any, 0, len(values))
for i, value := range values {
record, ok := value.(map[string]any)
if !ok {
return nil, fmt.Errorf("record %d is %T; merge inputs require object records", i+1, value)
}
records = append(records, record)
}
return records, nil
}
func encodeMergedRecords(records []map[string]any, format, sheet string, pretty bool) ([]byte, error) {
values := make([]any, len(records))
for i, record := range records {
values[i] = record
}
if !isTabular(format) {
return encodeStructured(values, format, pretty)
}
t := valueToTable(values)
if len(t.Header) == 0 && len(t.Rows) > 0 {
return nil, fmt.Errorf("merged records have no scalar fields to represent as a table")
}
return encodeTable(t, format, sheet)
}
func mergeRecords(primary, secondary []map[string]any, on string) ([]map[string]any, error) {
tokens, err := parsePath(on)
if err != nil {
return nil, fmt.Errorf("invalid merge path %q: %w", on, err)
}
lookup := make(map[string]map[string]any, len(secondary))
for i, record := range secondary {
value, err := valueAtPath(record, tokens)
if err != nil {
return nil, fmt.Errorf("merge record %d path %q: %w", i+1, on, err)
}
key, err := mergeKey(value)
if err != nil {
return nil, fmt.Errorf("merge record %d path %q: %w", i+1, on, err)
}
if _, exists := lookup[key]; exists {
return nil, fmt.Errorf("merge input has duplicate key %v at record %d", value, i+1)
}
lookup[key] = record
}
merged := make([]map[string]any, 0, len(primary))
for i, record := range primary {
value, err := valueAtPath(record, tokens)
if err != nil {
return nil, fmt.Errorf("primary record %d path %q: %w", i+1, on, err)
}
key, err := mergeKey(value)
if err != nil {
return nil, fmt.Errorf("primary record %d path %q: %w", i+1, on, err)
}
result := cloneMap(record)
if addition, ok := lookup[key]; ok {
if err := mergeMap(result, addition, ""); err != nil {
return nil, fmt.Errorf("primary record %d: %w", i+1, err)
}
}
merged = append(merged, result)
}
return merged, nil
}
func valueAtPath(root map[string]any, tokens []pathToken) (any, error) {
var current any = root
for _, token := range tokens {
if token.isIndex {
values, ok := current.([]any)
if !ok {
return nil, fmt.Errorf("expected an array at index [%d], found %T", token.index, current)
}
if token.index >= len(values) {
return nil, fmt.Errorf("array index [%d] does not exist", token.index)
}
current = values[token.index]
continue
}
object, ok := current.(map[string]any)
if !ok {
return nil, fmt.Errorf("expected an object before key %q, found %T", token.key, current)
}
value, exists := object[token.key]
if !exists {
return nil, fmt.Errorf("key %q does not exist", token.key)
}
current = value
}
return current, nil
}
func mergeKey(value any) (string, error) {
switch typed := value.(type) {
case string:
return "string:" + typed, nil
case bool:
return "bool:" + strconv.FormatBool(typed), nil
case json.Number:
return rationalNumberKey(typed.String())
case int:
return rationalNumberKey(strconv.FormatInt(int64(typed), 10))
case int8:
return rationalNumberKey(strconv.FormatInt(int64(typed), 10))
case int16:
return rationalNumberKey(strconv.FormatInt(int64(typed), 10))
case int32:
return rationalNumberKey(strconv.FormatInt(int64(typed), 10))
case int64:
return rationalNumberKey(strconv.FormatInt(typed, 10))
case uint:
return rationalNumberKey(strconv.FormatUint(uint64(typed), 10))
case uint8:
return rationalNumberKey(strconv.FormatUint(uint64(typed), 10))
case uint16:
return rationalNumberKey(strconv.FormatUint(uint64(typed), 10))
case uint32:
return rationalNumberKey(strconv.FormatUint(uint64(typed), 10))
case uint64:
return rationalNumberKey(strconv.FormatUint(typed, 10))
case float32:
return finiteFloatKey(float64(typed))
case float64:
return finiteFloatKey(typed)
case nil:
return "", fmt.Errorf("merge key is null")
default:
return "", fmt.Errorf("merge key must be a string, number, or boolean, found %T", value)
}
}
func finiteFloatKey(value float64) (string, error) {
if math.IsNaN(value) || math.IsInf(value, 0) {
return "", fmt.Errorf("merge key must be a finite number")
}
return rationalNumberKey(strconv.FormatFloat(value, 'g', -1, 64))
}
func rationalNumberKey(value string) (string, error) {
rational, ok := new(big.Rat).SetString(value)
if !ok {
return "", fmt.Errorf("value %q is not a valid number", value)
}
return "number:" + rational.RatString(), nil
}
func mergeMap(destination, source map[string]any, prefix string) error {
for key, incoming := range source {
path := appendPathKey(prefix, key)
existing, exists := destination[key]
if !exists {
destination[key] = cloneValue(incoming)
continue
}
existingMap, existingIsMap := existing.(map[string]any)
incomingMap, incomingIsMap := incoming.(map[string]any)
if existingIsMap && incomingIsMap {
if err := mergeMap(existingMap, incomingMap, path); err != nil {
return err
}
continue
}
if !mergeValuesEqual(existing, incoming) {
return fmt.Errorf("merge field %q conflicts: primary=%v merge=%v", path, existing, incoming)
}
}
return nil
}
func mergeValuesEqual(first, second any) bool {
if reflect.DeepEqual(first, second) {
return true
}
firstKey, firstErr := mergeKey(first)
secondKey, secondErr := mergeKey(second)
return firstErr == nil && secondErr == nil && firstKey == secondKey
}
func cloneMap(value map[string]any) map[string]any {
cloned := make(map[string]any, len(value))
for key, child := range value {
cloned[key] = cloneValue(child)
}
return cloned
}
func cloneValue(value any) any {
switch typed := value.(type) {
case map[string]any:
return cloneMap(typed)
case []any:
cloned := make([]any, len(typed))
for i, child := range typed {
cloned[i] = cloneValue(child)
}
return cloned
default:
return typed
}
}
+134
View File
@@ -0,0 +1,134 @@
package main
import (
"bytes"
"encoding/json"
"strings"
"testing"
)
func TestMergeCLIEnrichesPrimaryRecords(t *testing.T) {
t.Parallel()
primary := "id,name\n1,Alice\n2,Bob\n3,Carol\n"
details := "[{\"id\":1,\"contact\":{\"email\":\"[email protected]\"},\"active\":true},{\"id\":2,\"contact\":{\"email\":\"[email protected]\"},\"active\":false},{\"id\":99,\"ignored\":\"no primary match\"}]"
detailsPath := t.TempDir() + "/details.json"
if err := writeOutput(detailsPath, ioDiscard{}, []byte(details)); err != nil {
t.Fatal(err)
}
var out bytes.Buffer
err := run(
[]string{"-from", "csv", "-to", "json", "-merge", detailsPath, "-on", "id", "-pretty=false"},
strings.NewReader(primary), &out, ioDiscard{},
)
if err != nil {
t.Fatal(err)
}
var got []map[string]any
if err := json.Unmarshal(out.Bytes(), &got); err != nil {
t.Fatal(err)
}
if len(got) != 3 {
t.Fatalf("record count = %d, want 3", len(got))
}
if got[0]["active"] != true || got[1]["active"] != false {
t.Fatalf("merged active fields = %#v, %#v", got[0]["active"], got[1]["active"])
}
if _, exists := got[2]["active"]; exists {
t.Fatalf("unmatched primary record was modified: %#v", got[2])
}
contact := got[0]["contact"].(map[string]any)
if contact["email"] != "[email protected]" {
t.Fatalf("nested merged field = %#v", contact)
}
}
func TestMergeRecordsSupportsNestedKeyAndFields(t *testing.T) {
t.Parallel()
primary := []map[string]any{{"user": map[string]any{"id": "001", "name": "Alice"}}}
secondary := []map[string]any{{"user": map[string]any{"id": "001", "role": "admin"}}}
got, err := mergeRecords(primary, secondary, "user.id")
if err != nil {
t.Fatal(err)
}
user := got[0]["user"].(map[string]any)
if user["name"] != "Alice" || user["role"] != "admin" {
t.Fatalf("merged user = %#v", user)
}
if _, exists := primary[0]["user"].(map[string]any)["role"]; exists {
t.Fatal("merge mutated primary input")
}
}
func TestMergeRecordsRejectsDuplicateLookupKey(t *testing.T) {
t.Parallel()
_, err := mergeRecords(
[]map[string]any{{"id": int64(1)}},
[]map[string]any{{"id": int64(1)}, {"id": int64(1)}},
"id",
)
if err == nil || !strings.Contains(err.Error(), "duplicate key") {
t.Fatalf("error = %v, want duplicate-key error", err)
}
}
func TestMergeRecordsRejectsConflictingField(t *testing.T) {
t.Parallel()
_, err := mergeRecords(
[]map[string]any{{"id": int64(1), "name": "Alice"}},
[]map[string]any{{"id": int64(1), "name": "Alicia"}},
"id",
)
if err == nil || !strings.Contains(err.Error(), "merge field \"name\" conflicts") {
t.Fatalf("error = %v, want field-conflict error", err)
}
}
func TestMergeRecordsMatchesNumericKeysAcrossInputTypes(t *testing.T) {
t.Parallel()
got, err := mergeRecords(
[]map[string]any{{"id": int64(1)}},
[]map[string]any{{"id": json.Number("1.0"), "label": "one"}},
"id",
)
if err != nil {
t.Fatal(err)
}
if got[0]["label"] != "one" {
t.Fatalf("merged record = %#v", got[0])
}
}
func TestMergeRecordsPreservesLargeNumericKeyPrecision(t *testing.T) {
t.Parallel()
got, err := mergeRecords(
[]map[string]any{{"id": uint64(18_446_744_073_709_551_615)}},
[]map[string]any{{"id": json.Number("18446744073709551615"), "label": "max"}},
"id",
)
if err != nil {
t.Fatal(err)
}
if got[0]["label"] != "max" {
t.Fatalf("merged record = %#v", got[0])
}
}
func TestMergeOptionsRequireJoinPathAndFormat(t *testing.T) {
t.Parallel()
opt, err := parseOptions([]string{"-from", "csv", "-to", "json", "-merge", "-"}, ioDiscard{})
if err != nil {
t.Fatal(err)
}
if err := opt.resolveFormats(); err == nil || !strings.Contains(err.Error(), "-merge-from is required") {
t.Fatalf("error = %v, want merge-format error", err)
}
opt, err = parseOptions([]string{"-from", "csv", "-to", "json", "-merge", "lookup.json"}, ioDiscard{})
if err != nil {
t.Fatal(err)
}
if err := opt.resolveFormats(); err == nil || !strings.Contains(err.Error(), "-on is required") {
t.Fatalf("error = %v, want join-path error", err)
}
}