6 Commits
Author SHA1 Message Date
agent 5a8f7374ea Merge pull request 'キー指定によるデータマージ機能を追加' (#10) from feature/data-merge-9 into main
CI / test (push) Successful in 14s
2026-10-26 17:51:48 +09:00
agent 0634755433 fix: update vulnerable Excelize dependency
CI / test (pull_request) Successful in 50s
2026-10-26 16:05:18 +09:00
agent d3a08c25e2 feat: add keyed record merge (#9)
CI / test (pull_request) Failing after 39s
2026-10-26 15:55:49 +09:00
kamat 4c524e7a24 Merge pull request 'Markdownテーブル形式に対応' (#8) from feature/markdown-table-format into main
CI / test (push) Successful in 11s
Release / release (push) Successful in 1m54s
Reviewed-on: #8
2026-09-05 16:59:36 +09:00
agent 83c2c9e59f Require patched Go 1.25 release
CI / test (pull_request) Successful in 45s
2026-09-05 16:52:28 +09:00
agent 60a43a42e5 Add Markdown table format support
CI / test (pull_request) Failing after 1m6s
2026-09-05 16:45:44 +09:00
14 changed files with 793 additions and 49 deletions
+46 -3
View File
@@ -15,10 +15,11 @@ TSVへ変換したり、Excelからコピーした表を再び構造化データ
- CSV
- TSV
- XLSX
- Markdown table (`markdown` / `md`)
## インストール
Go 1.25以上が必要です。
Go 1.25.13以上が必要です。
```sh
go install git.rumginger.org/agent/dataxl/cmd/dataxl@latest
@@ -55,6 +56,12 @@ JSONからExcel workbookを作成:
dataxl -from json -to xlsx -i people.json -o people.xlsx
```
Markdownのテーブルを作成:
```sh
dataxl -from yaml -to markdown -i examples/people.yaml -o people.md
```
Excel workbookをYAMLへ変換:
```sh
@@ -69,6 +76,30 @@ dataxl -i examples/people.yaml -o people.tsv
dataxl -from tsv -to yaml < people.tsv
```
## データのマージ
主入力の各レコードを保持したまま、同じキーを持つ第2入力のフィールドを追加できます。
次の例は `people.csv` の `id` と `details.json` の `id` を照合し、JSON側の
フィールドを追加してYAMLへ出力します。
```sh
dataxl -i people.csv -merge details.json -on id -to yaml
```
マージはleft joinです。主入力にだけ存在するレコードはそのまま残り、第2入力にだけ
存在するレコードは出力しません。`-on` には `user.id` や
`items[0].sku` のような列パスも指定できます。
データを暗黙に上書きしないため、次の場合はエラーになります。
- いずれかのレコードにマージキーがない、またはキーがscalarでない
- 第2入力に同じキーが複数ある
- 同名フィールドの値が異なる
入力形式は各ファイル名から推定します。第2入力がstdinや拡張子なしの場合は
`-merge-from` を指定してください。主入力と第2入力を同時にstdinから読むことは
できません。
## Excelとの連携
Excelへ貼り付ける場合はTSVが便利です。
@@ -86,6 +117,15 @@ Excelから戻す場合は、シート上の範囲をコピーしてstdinへ渡
dataxl -from tsv -to yaml > restored.yaml
```
## Markdownテーブル
GitHub Flavored Markdown互換のヘッダー・区切り行を持つテーブルを読み書きできます。
外側の `|` は省略可能で、区切り行の `:` による左寄せ・中央寄せ・右寄せ指定も
入力時に受け付けます(配置指定自体は保持しません)。
セル内の `|` とバックスラッシュはMarkdownのbackslash escapeを使用します。
改行や前後の空白・タブはHTML文字参照として表し、表形式間の変換で保持します。
## 入れ子構造の表現
構造化データの入れ子は、Excelで編集しやすい列名へ展開されます。
@@ -154,10 +194,13 @@ CSV、TSV、XLSXから構造化形式へ戻す際は、セル文字列を次の
- `-i`: 入力ファイル。省略時はstdin。
- `-o`: 出力ファイル。省略時はstdout。
- `-from`: 入力形式。`json`, `yaml`, `toml`, `csv`, `tsv`, `xlsx`。
- `-to`: 出力形式。`json`, `yaml`, `toml`, `csv`, `tsv`, `xlsx`。
- `-from`: 入力形式。`json`, `yaml`, `toml`, `csv`, `tsv`, `xlsx`, `markdown`。
- `-to`: 出力形式。`json`, `yaml`, `toml`, `csv`, `tsv`, `xlsx`, `markdown`。
- `-sheet`: XLSXの読み書きに使うシート名。既定値は `Sheet1`。
- `-pretty`: JSONなどの構造化出力を整形するか。既定値は `true`。
- `-merge`: 主入力へフィールドを追加する第2入力ファイル。
- `-merge-from`: 第2入力の形式。通常は `-merge` の拡張子から推定。
- `-on`: マージ時に両入力を照合する列パス。
- `-version`: バージョンを表示して終了します。
## 現在の制約
+4 -2
View File
@@ -51,6 +51,8 @@ func normalizeFormat(format string) string {
switch format {
case "yml":
return "yaml"
case "md", "mdown", "mkd":
return "markdown"
case "xlsm", "xls":
return "xlsx"
default:
@@ -67,7 +69,7 @@ func inferFormat(path string) string {
func validateFormat(format string) error {
switch format {
case "json", "yaml", "toml", "csv", "tsv", "xlsx":
case "json", "yaml", "toml", "csv", "tsv", "xlsx", "markdown":
return nil
default:
return fmt.Errorf("unsupported format %q", format)
@@ -75,5 +77,5 @@ func validateFormat(format string) error {
}
func isTabular(format string) bool {
return format == "csv" || format == "tsv" || format == "xlsx"
return format == "csv" || format == "tsv" || format == "xlsx" || format == "markdown"
}
+1 -1
View File
@@ -6,7 +6,7 @@ import (
)
func TestConversionMatrixPreservesRepresentativeTable(t *testing.T) {
formats := []string{"json", "yaml", "toml", "csv", "tsv", "xlsx"}
formats := []string{"json", "yaml", "toml", "csv", "tsv", "xlsx", "markdown"}
records := []any{
map[string]any{
"active": true,
+2 -2
View File
@@ -220,7 +220,7 @@ func TestDelimitedSingleEmptyCellRowRoundTrip(t *testing.T) {
func TestStructuredRecordsWithoutScalarFieldsAreRejectedForTables(t *testing.T) {
for _, input := range []string{`{}`, `[{}]`} {
for _, format := range []string{"csv", "tsv", "xlsx"} {
for _, format := range []string{"csv", "tsv", "xlsx", "markdown"} {
t.Run(format+"_"+input, func(t *testing.T) {
_, err := convert([]byte(input), "json", format, "Sheet1", false)
if err == nil || !strings.Contains(err.Error(), "no scalar fields") {
@@ -232,7 +232,7 @@ func TestStructuredRecordsWithoutScalarFieldsAreRejectedForTables(t *testing.T)
}
func TestEmptyRecordListCanRoundTripThroughDelimitedFormats(t *testing.T) {
for _, format := range []string{"csv", "tsv"} {
for _, format := range []string{"csv", "tsv", "markdown"} {
t.Run(format, func(t *testing.T) {
tabular, err := convert([]byte(`[]`), "json", format, "Sheet1", false)
if err != nil {
+1 -1
View File
@@ -1,7 +1,7 @@
// Package main implements the dataxl command-line converter.
//
// Conversion uses two internal representations: structured Go values for
// JSON/YAML/TOML and table for CSV/TSV/XLSX. Nested structured values cross the
// JSON/YAML/TOML and table for CSV/TSV/XLSX/Markdown. Nested structured values cross the
// boundary through dotted and indexed column paths such as user.name and
// items[0].sku.
package main
+60 -12
View File
@@ -14,13 +14,16 @@ var buildVersion = "dev"
// options contains CLI concerns only. Conversion functions receive explicit
// arguments so they can be reused and tested without constructing a FlagSet.
type options struct {
inFile string
outFile string
from string
to string
sheet string
pretty bool
version bool
inFile string
outFile string
from string
to string
sheet string
pretty bool
version bool
mergeFile string
mergeFrom string
mergeOn string
}
func main() {
@@ -47,7 +50,22 @@ func run(args []string, stdin io.Reader, stdout, stderr io.Writer) error {
if err != nil {
return err
}
output, err := convert(input, opt.from, opt.to, opt.sheet, opt.pretty)
var output []byte
if opt.mergeFile != "" {
if isStdinPath(opt.inFile) && isStdinPath(opt.mergeFile) {
return fmt.Errorf("primary and merge inputs cannot both use stdin")
}
mergeInput, readErr := readInput(opt.mergeFile, stdin)
if readErr != nil {
return readErr
}
output, err = mergeAndConvert(
input, opt.from, mergeInput, opt.mergeFrom, opt.mergeOn,
opt.to, opt.sheet, opt.pretty,
)
} else {
output, err = convert(input, opt.from, opt.to, opt.sheet, opt.pretty)
}
if err != nil {
return err
}
@@ -60,19 +78,23 @@ func parseOptions(args []string, stderr io.Writer) (options, error) {
fs.SetOutput(stderr)
fs.StringVar(&opt.inFile, "i", "", "input file, defaults to stdin")
fs.StringVar(&opt.outFile, "o", "", "output file, defaults to stdout")
fs.StringVar(&opt.from, "from", "", "input format: json, yaml, toml, csv, tsv, xlsx")
fs.StringVar(&opt.to, "to", "", "output format: json, yaml, toml, csv, tsv, xlsx")
fs.StringVar(&opt.from, "from", "", "input format: json, yaml, toml, csv, tsv, xlsx, markdown")
fs.StringVar(&opt.to, "to", "", "output format: json, yaml, toml, csv, tsv, xlsx, markdown")
fs.StringVar(&opt.sheet, "sheet", "Sheet1", "worksheet name for xlsx input/output")
fs.BoolVar(&opt.pretty, "pretty", true, "pretty-print structured output")
fs.BoolVar(&opt.version, "version", false, "print version and exit")
fs.StringVar(&opt.mergeFile, "merge", "", "second input file used to enrich primary records")
fs.StringVar(&opt.mergeFrom, "merge-from", "", "second input format; inferred from -merge when omitted")
fs.StringVar(&opt.mergeOn, "on", "", "column path used to match records when -merge is set")
fs.Usage = func() {
_, _ = fmt.Fprintln(stderr, `Usage:
dataxl -from yaml -to tsv -i input.yaml -o output.tsv
dataxl -from tsv -to json < paste-from-excel.tsv
dataxl -from json -to xlsx -i data.json -o data.xlsx
dataxl -i people.csv -merge details.json -on id -to yaml
Formats:
json, yaml/yml, toml, csv, tsv, xlsx
json, yaml/yml, toml, csv, tsv, xlsx, markdown/md
Notes:
Structured records are flattened into spreadsheet columns such as user.name
@@ -102,6 +124,7 @@ func resolvedVersion() string {
func (opt *options) resolveFormats() error {
opt.from = normalizeFormat(opt.from)
opt.to = normalizeFormat(opt.to)
opt.mergeFrom = normalizeFormat(opt.mergeFrom)
if opt.from == "" {
opt.from = inferFormat(opt.inFile)
}
@@ -114,7 +137,32 @@ func (opt *options) resolveFormats() error {
if err := validateFormat(opt.from); err != nil {
return err
}
return validateFormat(opt.to)
if err := validateFormat(opt.to); err != nil {
return err
}
if opt.mergeFile == "" {
if opt.mergeFrom != "" || opt.mergeOn != "" {
return fmt.Errorf("-merge-from and -on require -merge")
}
return nil
}
if opt.mergeFrom == "" {
opt.mergeFrom = inferFormat(opt.mergeFile)
}
if opt.mergeFrom == "" {
return fmt.Errorf("-merge-from is required when the second input format cannot be inferred")
}
if err := validateFormat(opt.mergeFrom); err != nil {
return fmt.Errorf("merge input: %w", err)
}
if strings.TrimSpace(opt.mergeOn) == "" {
return fmt.Errorf("-on is required when -merge is set")
}
return nil
}
func isStdinPath(path string) bool {
return path == "" || path == "-"
}
func readInput(path string, stdin io.Reader) ([]byte, error) {
+43
View File
@@ -129,6 +129,49 @@ func TestResolveFormatsFromFileExtensions(t *testing.T) {
}
}
func TestResolveMarkdownFormatFromFileExtension(t *testing.T) {
opt := options{inFile: "input.json", outFile: "output.md"}
if err := opt.resolveFormats(); err != nil {
t.Fatal(err)
}
if opt.from != "json" || opt.to != "markdown" {
t.Fatalf("resolved formats = %q -> %q, want json -> markdown", opt.from, opt.to)
}
}
func TestMarkdownTableRoundTrip(t *testing.T) {
want := table{
Header: []string{"id", "name", "note"},
Rows: [][]string{{"1", "Alice | Bob", "line 1\nline 2\\n"}},
}
encoded, err := encodeTable(want, "markdown", "")
if err != nil {
t.Fatal(err)
}
got, err := parseTable(encoded, "markdown", "")
if err != nil {
t.Fatalf("parse generated Markdown: %v\n%s", err, encoded)
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("table = %#v, want %#v\n%s", got, want, encoded)
}
}
func TestMarkdownTableAcceptsAlignmentAndOptionalOuterPipes(t *testing.T) {
input := []byte("id | name | note\n---: | :--- | :---:\n1 | Alice \\| Bob | ok\n")
got, err := parseTable(input, "markdown", "")
if err != nil {
t.Fatal(err)
}
want := table{
Header: []string{"id", "name", "note"},
Rows: [][]string{{"1", "Alice | Bob", "ok"}},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("table = %#v, want %#v", got, want)
}
}
func TestVersionFlagDoesNotRequireFormats(t *testing.T) {
oldVersion := buildVersion
buildVersion = "v9.8.7-test"
+255
View File
@@ -0,0 +1,255 @@
package main
import (
"encoding/json"
"fmt"
"math"
"math/big"
"reflect"
"strconv"
)
// mergeAndConvert parses two record sets, enriches primary records with fields
// from matching secondary records, and encodes the result in the requested
// output format. Primary order and unmatched primary records are preserved.
func mergeAndConvert(
primary []byte,
primaryFormat string,
secondary []byte,
secondaryFormat string,
on string,
to string,
sheet string,
pretty bool,
) ([]byte, error) {
primaryRecords, err := parseMergeRecords(primary, primaryFormat, sheet)
if err != nil {
return nil, fmt.Errorf("primary input: %w", err)
}
secondaryRecords, err := parseMergeRecords(secondary, secondaryFormat, sheet)
if err != nil {
return nil, fmt.Errorf("merge input: %w", err)
}
merged, err := mergeRecords(primaryRecords, secondaryRecords, on)
if err != nil {
return nil, err
}
return encodeMergedRecords(merged, to, sheet, pretty)
}
func parseMergeRecords(input []byte, format, sheet string) ([]map[string]any, error) {
if isTabular(format) {
t, err := parseTable(input, format, sheet)
if err != nil {
return nil, err
}
return tableToRecords(t)
}
value, err := parseStructured(input, format)
if err != nil {
return nil, err
}
values := recordsFromValue(value)
records := make([]map[string]any, 0, len(values))
for i, value := range values {
record, ok := value.(map[string]any)
if !ok {
return nil, fmt.Errorf("record %d is %T; merge inputs require object records", i+1, value)
}
records = append(records, record)
}
return records, nil
}
func encodeMergedRecords(records []map[string]any, format, sheet string, pretty bool) ([]byte, error) {
values := make([]any, len(records))
for i, record := range records {
values[i] = record
}
if !isTabular(format) {
return encodeStructured(values, format, pretty)
}
t := valueToTable(values)
if len(t.Header) == 0 && len(t.Rows) > 0 {
return nil, fmt.Errorf("merged records have no scalar fields to represent as a table")
}
return encodeTable(t, format, sheet)
}
func mergeRecords(primary, secondary []map[string]any, on string) ([]map[string]any, error) {
tokens, err := parsePath(on)
if err != nil {
return nil, fmt.Errorf("invalid merge path %q: %w", on, err)
}
lookup := make(map[string]map[string]any, len(secondary))
for i, record := range secondary {
value, err := valueAtPath(record, tokens)
if err != nil {
return nil, fmt.Errorf("merge record %d path %q: %w", i+1, on, err)
}
key, err := mergeKey(value)
if err != nil {
return nil, fmt.Errorf("merge record %d path %q: %w", i+1, on, err)
}
if _, exists := lookup[key]; exists {
return nil, fmt.Errorf("merge input has duplicate key %v at record %d", value, i+1)
}
lookup[key] = record
}
merged := make([]map[string]any, 0, len(primary))
for i, record := range primary {
value, err := valueAtPath(record, tokens)
if err != nil {
return nil, fmt.Errorf("primary record %d path %q: %w", i+1, on, err)
}
key, err := mergeKey(value)
if err != nil {
return nil, fmt.Errorf("primary record %d path %q: %w", i+1, on, err)
}
result := cloneMap(record)
if addition, ok := lookup[key]; ok {
if err := mergeMap(result, addition, ""); err != nil {
return nil, fmt.Errorf("primary record %d: %w", i+1, err)
}
}
merged = append(merged, result)
}
return merged, nil
}
func valueAtPath(root map[string]any, tokens []pathToken) (any, error) {
var current any = root
for _, token := range tokens {
if token.isIndex {
values, ok := current.([]any)
if !ok {
return nil, fmt.Errorf("expected an array at index [%d], found %T", token.index, current)
}
if token.index >= len(values) {
return nil, fmt.Errorf("array index [%d] does not exist", token.index)
}
current = values[token.index]
continue
}
object, ok := current.(map[string]any)
if !ok {
return nil, fmt.Errorf("expected an object before key %q, found %T", token.key, current)
}
value, exists := object[token.key]
if !exists {
return nil, fmt.Errorf("key %q does not exist", token.key)
}
current = value
}
return current, nil
}
func mergeKey(value any) (string, error) {
switch typed := value.(type) {
case string:
return "string:" + typed, nil
case bool:
return "bool:" + strconv.FormatBool(typed), nil
case json.Number:
return rationalNumberKey(typed.String())
case int:
return rationalNumberKey(strconv.FormatInt(int64(typed), 10))
case int8:
return rationalNumberKey(strconv.FormatInt(int64(typed), 10))
case int16:
return rationalNumberKey(strconv.FormatInt(int64(typed), 10))
case int32:
return rationalNumberKey(strconv.FormatInt(int64(typed), 10))
case int64:
return rationalNumberKey(strconv.FormatInt(typed, 10))
case uint:
return rationalNumberKey(strconv.FormatUint(uint64(typed), 10))
case uint8:
return rationalNumberKey(strconv.FormatUint(uint64(typed), 10))
case uint16:
return rationalNumberKey(strconv.FormatUint(uint64(typed), 10))
case uint32:
return rationalNumberKey(strconv.FormatUint(uint64(typed), 10))
case uint64:
return rationalNumberKey(strconv.FormatUint(typed, 10))
case float32:
return finiteFloatKey(float64(typed))
case float64:
return finiteFloatKey(typed)
case nil:
return "", fmt.Errorf("merge key is null")
default:
return "", fmt.Errorf("merge key must be a string, number, or boolean, found %T", value)
}
}
func finiteFloatKey(value float64) (string, error) {
if math.IsNaN(value) || math.IsInf(value, 0) {
return "", fmt.Errorf("merge key must be a finite number")
}
return rationalNumberKey(strconv.FormatFloat(value, 'g', -1, 64))
}
func rationalNumberKey(value string) (string, error) {
rational, ok := new(big.Rat).SetString(value)
if !ok {
return "", fmt.Errorf("value %q is not a valid number", value)
}
return "number:" + rational.RatString(), nil
}
func mergeMap(destination, source map[string]any, prefix string) error {
for key, incoming := range source {
path := appendPathKey(prefix, key)
existing, exists := destination[key]
if !exists {
destination[key] = cloneValue(incoming)
continue
}
existingMap, existingIsMap := existing.(map[string]any)
incomingMap, incomingIsMap := incoming.(map[string]any)
if existingIsMap && incomingIsMap {
if err := mergeMap(existingMap, incomingMap, path); err != nil {
return err
}
continue
}
if !mergeValuesEqual(existing, incoming) {
return fmt.Errorf("merge field %q conflicts: primary=%v merge=%v", path, existing, incoming)
}
}
return nil
}
func mergeValuesEqual(first, second any) bool {
if reflect.DeepEqual(first, second) {
return true
}
firstKey, firstErr := mergeKey(first)
secondKey, secondErr := mergeKey(second)
return firstErr == nil && secondErr == nil && firstKey == secondKey
}
func cloneMap(value map[string]any) map[string]any {
cloned := make(map[string]any, len(value))
for key, child := range value {
cloned[key] = cloneValue(child)
}
return cloned
}
func cloneValue(value any) any {
switch typed := value.(type) {
case map[string]any:
return cloneMap(typed)
case []any:
cloned := make([]any, len(typed))
for i, child := range typed {
cloned[i] = cloneValue(child)
}
return cloned
default:
return typed
}
}
+134
View File
@@ -0,0 +1,134 @@
package main
import (
"bytes"
"encoding/json"
"strings"
"testing"
)
func TestMergeCLIEnrichesPrimaryRecords(t *testing.T) {
t.Parallel()
primary := "id,name\n1,Alice\n2,Bob\n3,Carol\n"
details := "[{\"id\":1,\"contact\":{\"email\":\"[email protected]\"},\"active\":true},{\"id\":2,\"contact\":{\"email\":\"[email protected]\"},\"active\":false},{\"id\":99,\"ignored\":\"no primary match\"}]"
detailsPath := t.TempDir() + "/details.json"
if err := writeOutput(detailsPath, ioDiscard{}, []byte(details)); err != nil {
t.Fatal(err)
}
var out bytes.Buffer
err := run(
[]string{"-from", "csv", "-to", "json", "-merge", detailsPath, "-on", "id", "-pretty=false"},
strings.NewReader(primary), &out, ioDiscard{},
)
if err != nil {
t.Fatal(err)
}
var got []map[string]any
if err := json.Unmarshal(out.Bytes(), &got); err != nil {
t.Fatal(err)
}
if len(got) != 3 {
t.Fatalf("record count = %d, want 3", len(got))
}
if got[0]["active"] != true || got[1]["active"] != false {
t.Fatalf("merged active fields = %#v, %#v", got[0]["active"], got[1]["active"])
}
if _, exists := got[2]["active"]; exists {
t.Fatalf("unmatched primary record was modified: %#v", got[2])
}
contact := got[0]["contact"].(map[string]any)
if contact["email"] != "[email protected]" {
t.Fatalf("nested merged field = %#v", contact)
}
}
func TestMergeRecordsSupportsNestedKeyAndFields(t *testing.T) {
t.Parallel()
primary := []map[string]any{{"user": map[string]any{"id": "001", "name": "Alice"}}}
secondary := []map[string]any{{"user": map[string]any{"id": "001", "role": "admin"}}}
got, err := mergeRecords(primary, secondary, "user.id")
if err != nil {
t.Fatal(err)
}
user := got[0]["user"].(map[string]any)
if user["name"] != "Alice" || user["role"] != "admin" {
t.Fatalf("merged user = %#v", user)
}
if _, exists := primary[0]["user"].(map[string]any)["role"]; exists {
t.Fatal("merge mutated primary input")
}
}
func TestMergeRecordsRejectsDuplicateLookupKey(t *testing.T) {
t.Parallel()
_, err := mergeRecords(
[]map[string]any{{"id": int64(1)}},
[]map[string]any{{"id": int64(1)}, {"id": int64(1)}},
"id",
)
if err == nil || !strings.Contains(err.Error(), "duplicate key") {
t.Fatalf("error = %v, want duplicate-key error", err)
}
}
func TestMergeRecordsRejectsConflictingField(t *testing.T) {
t.Parallel()
_, err := mergeRecords(
[]map[string]any{{"id": int64(1), "name": "Alice"}},
[]map[string]any{{"id": int64(1), "name": "Alicia"}},
"id",
)
if err == nil || !strings.Contains(err.Error(), "merge field \"name\" conflicts") {
t.Fatalf("error = %v, want field-conflict error", err)
}
}
func TestMergeRecordsMatchesNumericKeysAcrossInputTypes(t *testing.T) {
t.Parallel()
got, err := mergeRecords(
[]map[string]any{{"id": int64(1)}},
[]map[string]any{{"id": json.Number("1.0"), "label": "one"}},
"id",
)
if err != nil {
t.Fatal(err)
}
if got[0]["label"] != "one" {
t.Fatalf("merged record = %#v", got[0])
}
}
func TestMergeRecordsPreservesLargeNumericKeyPrecision(t *testing.T) {
t.Parallel()
got, err := mergeRecords(
[]map[string]any{{"id": uint64(18_446_744_073_709_551_615)}},
[]map[string]any{{"id": json.Number("18446744073709551615"), "label": "max"}},
"id",
)
if err != nil {
t.Fatal(err)
}
if got[0]["label"] != "max" {
t.Fatalf("merged record = %#v", got[0])
}
}
func TestMergeOptionsRequireJoinPathAndFormat(t *testing.T) {
t.Parallel()
opt, err := parseOptions([]string{"-from", "csv", "-to", "json", "-merge", "-"}, ioDiscard{})
if err != nil {
t.Fatal(err)
}
if err := opt.resolveFormats(); err == nil || !strings.Contains(err.Error(), "-merge-from is required") {
t.Fatalf("error = %v, want merge-format error", err)
}
opt, err = parseOptions([]string{"-from", "csv", "-to", "json", "-merge", "lookup.json"}, ioDiscard{})
if err != nil {
t.Fatal(err)
}
if err := opt.resolveFormats(); err == nil || !strings.Contains(err.Error(), "-on is required") {
t.Fatalf("error = %v, want join-path error", err)
}
}
+200 -1
View File
@@ -5,14 +5,16 @@ import (
"encoding/csv"
"encoding/json"
"fmt"
"html"
"sort"
"strconv"
"strings"
"unicode/utf8"
"github.com/xuri/excelize/v2"
)
// table is the common representation for CSV, TSV and XLSX. Rows are padded
// table is the common representation for CSV, TSV, XLSX and Markdown. Rows are padded
// to Header width when read, which keeps subsequent conversions rectangular.
type table struct {
Header []string
@@ -36,6 +38,8 @@ func parseTable(input []byte, format, sheet string) (table, error) {
return table{}, err
}
return rowsToTable(rows)
case "markdown":
return readMarkdown(input)
default:
return table{}, fmt.Errorf("format %q is not tabular", format)
}
@@ -93,11 +97,206 @@ func encodeTable(t table, format, sheet string) ([]byte, error) {
return writeDelimited(t, '\t')
case "xlsx":
return writeXLSX(t, sheet)
case "markdown":
return writeMarkdown(t)
default:
return nil, fmt.Errorf("format %q is not tabular", format)
}
}
func readMarkdown(input []byte) (table, error) {
input = bytes.TrimPrefix(input, []byte{0xEF, 0xBB, 0xBF})
if !utf8.Valid(input) {
return table{}, fmt.Errorf("markdown input is not valid UTF-8")
}
text := strings.ReplaceAll(string(input), "\r\n", "\n")
text = strings.TrimSpace(text)
if text == "" {
return table{}, nil
}
lines := strings.Split(text, "\n")
if len(lines) < 2 {
return table{}, fmt.Errorf("markdown table requires a header and separator row")
}
header, err := parseMarkdownRow(lines[0])
if err != nil {
return table{}, fmt.Errorf("markdown header: %w", err)
}
separator, err := parseMarkdownRow(lines[1])
if err != nil {
return table{}, fmt.Errorf("markdown separator: %w", err)
}
if len(separator) != len(header) {
return table{}, fmt.Errorf("markdown separator has %d fields but the header has %d", len(separator), len(header))
}
for i, cell := range separator {
value := strings.TrimSpace(cell)
value = strings.TrimPrefix(value, ":")
value = strings.TrimSuffix(value, ":")
if len(value) < 3 || strings.Trim(value, "-") != "" {
return table{}, fmt.Errorf("markdown separator field %d is invalid", i+1)
}
}
rows := make([][]string, 0, len(lines)-2)
for i, line := range lines[2:] {
if strings.TrimSpace(line) == "" {
continue
}
row, err := parseMarkdownRow(line)
if err != nil {
return table{}, fmt.Errorf("markdown row %d: %w", i+3, err)
}
if len(row) > len(header) {
return table{}, fmt.Errorf("row %d has %d fields but the header has %d", i+3, len(row), len(header))
}
rows = append(rows, padRow(row, len(header)))
}
return table{Header: header, Rows: rows}, nil
}
func parseMarkdownRow(line string) ([]string, error) {
line = strings.TrimSpace(strings.TrimSuffix(line, "\r"))
if strings.HasPrefix(line, "|") {
line = line[1:]
}
if hasUnescapedTrailingPipe(line) {
line = strings.TrimSpace(line[:len(line)-1])
}
var cells []string
var cell strings.Builder
escaped := false
for _, r := range line {
if escaped {
cell.WriteRune('\\')
cell.WriteRune(r)
escaped = false
continue
}
if r == '\\' {
escaped = true
continue
}
if r == '|' {
cells = append(cells, decodeMarkdownCell(strings.Trim(cell.String(), " \t")))
cell.Reset()
continue
}
cell.WriteRune(r)
}
if escaped {
return nil, fmt.Errorf("row ends with an incomplete escape")
}
cells = append(cells, decodeMarkdownCell(strings.Trim(cell.String(), " \t")))
return cells, nil
}
func decodeMarkdownCell(value string) string {
var b strings.Builder
escaped := false
for _, r := range value {
if escaped {
if r == '\\' || r == '|' {
b.WriteRune(r)
} else {
b.WriteRune('\\')
b.WriteRune(r)
}
escaped = false
continue
}
if r == '\\' {
escaped = true
continue
}
b.WriteRune(r)
}
return html.UnescapeString(b.String())
}
func hasUnescapedTrailingPipe(line string) bool {
line = strings.TrimSpace(line)
if !strings.HasSuffix(line, "|") {
return false
}
backslashes := 0
for i := len(line) - 2; i >= 0 && line[i] == '\\'; i-- {
backslashes++
}
return backslashes%2 == 0
}
func writeMarkdown(t table) ([]byte, error) {
if len(t.Header) == 0 && len(t.Rows) == 0 {
return nil, nil
}
if len(t.Header) == 0 {
return nil, fmt.Errorf("markdown table has rows but no header")
}
var b strings.Builder
writeRow := func(row []string) {
b.WriteString("| ")
for i, value := range row {
if i > 0 {
b.WriteString(" | ")
}
b.WriteString(escapeMarkdownCell(value))
}
b.WriteString(" |\n")
}
writeRow(t.Header)
b.WriteString("|")
for range t.Header {
b.WriteString(" --- |")
}
b.WriteByte('\n')
for _, row := range t.Rows {
writeRow(padRow(row, len(t.Header)))
}
return []byte(b.String()), nil
}
func escapeMarkdownCell(value string) string {
leadingEnd := len(value) - len(strings.TrimLeft(value, " \t"))
trailingStart := len(strings.TrimRight(value, " \t"))
var b strings.Builder
for i, r := range value {
switch r {
case '\\':
b.WriteString("\\\\")
case '|':
b.WriteString("\\|")
case '\r':
b.WriteString("&#13;")
case '\n':
b.WriteString("&#10;")
case ' ':
if i < leadingEnd || i >= trailingStart {
b.WriteString("&#32;")
} else {
b.WriteByte(' ')
}
case '\t':
if i < leadingEnd || i >= trailingStart {
b.WriteString("&#9;")
} else {
b.WriteByte('\t')
}
default:
switch r {
case '&':
b.WriteString("&amp;")
case '<':
b.WriteString("&lt;")
case '>':
b.WriteString("&gt;")
default:
b.WriteRune(r)
}
}
}
return b.String()
}
func writeDelimited(t table, comma rune) ([]byte, error) {
var b bytes.Buffer
w := csv.NewWriter(&b)
+23 -5
View File
@@ -3,7 +3,7 @@
`dataxl` は、ファイル形式ごとの差を小さくするために、内部表現を2つに分けています。
- structured value: JSON/YAML/TOMLから読める `map[string]any`, `[]any`, scalar
- table: Excel/CSV/TSVに近い `Header []string` と `Rows [][]string`
- table: Excel/CSV/TSV/Markdownに近い `Header []string` と `Rows [][]string`
変換は原則として次のどれかです。
@@ -19,8 +19,9 @@
- `main.go`: CLI option、stdin/stdout、ファイル入出力
- `conversion.go`: 変換経路の選択、形式名の正規化・推定
- `structured.go`: JSON/YAML/TOML adapter
- `table.go`: CSV/TSV/XLSX adapter、table model、flatten
- `table.go`: CSV/TSV/XLSX/Markdown adapter、table model、flatten
- `path.go`: セル値の型推定、列パスのparse、unflatten
- `merge.go`: record集合のparse、left join、競合検出、マージ結果のencode
`convert` はファイル入出力から独立しているため、CLIを経由せず変換matrixをテストできます。
形式固有処理は `parseStructured` / `encodeStructured` または
@@ -40,9 +41,12 @@ type table struct {
}
```
CSV/TSV/XLSXの読み込みでは、短い行を空文字で埋めて列数を揃えます。ヘッダーより
CSV/TSV/XLSX/Markdownの読み込みでは、短い行を空文字で埋めて列数を揃えます。ヘッダーより
長い行は、名前のない値を破棄しないようエラーにします。
XLSXの書き出しではヘッダーを太字にし、1行目を固定します。
Markdownでは区切り行のalignment markerを受け付けますが、table modelには保持しません。
セル内のdelimiterとバックスラッシュはbackslash escape、改行と前後空白はHTML文字参照で
可逆化します。
## Flattening
@@ -118,6 +122,20 @@ array-index = "[", digit, { digit }, "]";
- recordが存在するのにscalar fieldが1つもないstructured valueは、表へ変換すると
record数を失うため拒否します。空のrecord listは空の表として扱います。
## Record merge
`-merge` は主入力をleft side、第2入力をlookup sideとするleft joinです。両入力を
object recordの配列へ正規化し、`-on` の列パスからscalar keyを読み取ります。
主入力の順序と未一致recordを保持し、第2入力にしかないrecordは出力しません。
第2入力のkeyは一意でなければなりません。追加フィールドはmapを再帰的に統合しますが、
既存値と異なる同名フィールドは上書きせずエラーにします。数値keyおよび同名の数値値は、
CSV由来のintegerとJSON由来のnumberのように具体的なGo型が異なっても同じ数値なら一致と
みなします。文字列 `"1"` と数値 `1` は別keyです。
両入力を同じstdinから取得することはできません。片方だけstdinにする場合、もう片方は
ファイルで指定します。
## TOML Output
TOMLはトップレベル配列を直接表せないため、表からTOMLへ出力する場合など、
@@ -129,11 +147,11 @@ TOMLはトップレベル配列を直接表せないため、表からTOMLへ出
- `gopkg.in/yaml.v3`: YAML読み書き
- `github.com/BurntSushi/toml`: TOML読み書き
Go 1.25以上を前提にしています。
Go 1.25.13以上を前提にしています。
## Error handling
- 未対応形式、decode失敗、workbook/sheet操作失敗は呼び出し元へerrorを返します。
- CSV/TSVのinvalid UTF-8、headerより長い行、値を持つ空header列を拒否します。
- CSV/TSV/Markdownのinvalid UTF-8、headerより長い行、値を持つ空header列を拒否します。
- path復元中の重複header、構文エラー、型競合は変換全体をerrorにします。
- XLSXのstyle・pane設定も通常の変換errorとして扱い、不完全なworkbookを成功扱いしません。
+3 -1
View File
@@ -2,7 +2,7 @@
## Requirements
- Go 1.25 or later
- Go 1.25.13 or later
## Setup
@@ -58,6 +58,7 @@ The current tests cover:
- YAML -> TSV flattening
- TSV -> JSON path restoration
- JSON -> XLSX -> JSON round trip
- Markdown table parsing, escaping, and all-format conversion round trips
- structured -> structured conversion without CLI/file I/O
- extension normalization and format inference
- short table row padding and wider-row rejection
@@ -68,6 +69,7 @@ The current tests cover:
- wrapper arrays with sibling metadata
- duplicate, blank, malformed, and conflicting headers
- nested arrays and JSON-quoted path keys
- cross-format left join, nested merge paths, unmatched rows, duplicate keys, and field conflicts
When adding a new format or path rule, add tests around both directions where
possible.
+7 -7
View File
@@ -1,20 +1,20 @@
module git.rumginger.org/agent/dataxl
go 1.25.0
go 1.26.6
require (
github.com/BurntSushi/toml v1.6.0
github.com/xuri/excelize/v2 v2.11.0
github.com/xuri/excelize/v2 v2.11.1-0.20260930021559-01a9ff32fb3c
gopkg.in/yaml.v3 v3.0.1
)
require (
github.com/richardlehane/mscfb v1.0.7 // indirect
github.com/richardlehane/mscfb v1.0.8 // indirect
github.com/richardlehane/msoleps v1.0.6 // indirect
github.com/tiendc/go-deepcopy v1.7.2 // indirect
github.com/xuri/efp v0.0.1 // indirect
github.com/xuri/efp v0.0.2 // indirect
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect
golang.org/x/crypto v0.53.0 // indirect
golang.org/x/net v0.56.0 // indirect
golang.org/x/text v0.39.0 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/net v0.59.0 // indirect
golang.org/x/text v0.42.0 // indirect
)
+14 -14
View File
@@ -4,28 +4,28 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/richardlehane/mscfb v1.0.7 h1:oeoiM0WE79vHwE8RpIYYvIAc8ajTH2mb6UZm55/+EB0=
github.com/richardlehane/mscfb v1.0.7/go.mod h1:pe0+IUIc0AHh0+teNzBlJCtSyZdFOGgV4ZK9bsoV+Jo=
github.com/richardlehane/mscfb v1.0.8 h1:UXdg61fxF69/X9yMYuRHAWSrGXIul/UAPivAsUXMme8=
github.com/richardlehane/mscfb v1.0.8/go.mod h1:pe0+IUIc0AHh0+teNzBlJCtSyZdFOGgV4ZK9bsoV+Jo=
github.com/richardlehane/msoleps v1.0.6 h1:9BvkpjvD+iUBalUY4esMwv6uBkfOip/Lzvd93jvR9gg=
github.com/richardlehane/msoleps v1.0.6/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/tiendc/go-deepcopy v1.7.2 h1:Ut2yYR7W9tWjTQitganoIue4UGxZwCcJy3orjrrIj44=
github.com/tiendc/go-deepcopy v1.7.2/go.mod h1:4bKjNC2r7boYOkD2IOuZpYjmlDdzjbpTRyCx+goBCJQ=
github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8=
github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
github.com/xuri/excelize/v2 v2.11.0 h1:HxaEFl6sRN2+8J5a8HaKq+0M4FsjBGMnWWtjOCPSG88=
github.com/xuri/excelize/v2 v2.11.0/go.mod h1:jxFLbzaIwGQ5ufFNvYfUOHqXhfPaNmP14KWfmNz2Uak=
github.com/xuri/efp v0.0.2 h1:74yhBh0nfIWLOrN76wtShYVse94pADGYS0/c86jOp8Q=
github.com/xuri/efp v0.0.2/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
github.com/xuri/excelize/v2 v2.11.1-0.20260930021559-01a9ff32fb3c h1:aDSajGl9MFepc8NDZ7AUueR1/Zt8SZCVFe35hykA+xE=
github.com/xuri/excelize/v2 v2.11.1-0.20260930021559-01a9ff32fb3c/go.mod h1:d2KG/tW81cIhSeC3S7wENv3qRnOC3MDcrjDicKuZa6U=
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 h1:+C0TIdyyYmzadGaL/HBLbf3WdLgC29pgyhTjAT/0nuE=
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus=
golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/image v0.44.0 h1:+tDekMZED9+LrtB3G5xzRggpVh9CARjZqROla3R3R+I=
golang.org/x/image v0.44.0/go.mod h1:V8K3KE9KKKE+pLpQDOeN18w9oacNSvy1tDOirTu4xtY=
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=