laurentu
/
csv-parser
1
0
Fork 0
Code Pull Requests Projects Releases Activity

Algo ok, mais peut-être à optimiser ?

This commit is contained in:
laurentu 2024-08-08 11:02:10 +02:00
parent f92b89e01e
commit 367bd16e42
2 changed files with 140 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
csvparser.go
View File

@ -7,10 +7,12 @@ import (
) )
type CsvParser struct { type CsvParser struct {
enclosers []string enclosers []string
delimiters string delimiters string
mergeDelimiters string mergeDelimiters bool
fields []string fields []string
ignore string
maxFieldIndex int
} }
/* /*
@ -18,98 +20,117 @@ type CsvParser struct {
* mergeDelimiters: if true, all successive delimiters are considered as one * mergeDelimiters: if true, all successive delimiters are considered as one
* enclosers: array of string, each string contains a enclorser tuple: "", [], {}... * enclosers: array of string, each string contains a enclorser tuple: "", [], {}...
* lineFormat: format of the line (each fieldname or ignore) * lineFormat: format of the line (each fieldname or ignore)
*/ */
func (p *CsvParser) Initialize(delimiters string, mergeDelimiters bool, enclosers []string, lineFormat string) error { func (p *CsvParser) Initialize(delimiters string, mergeDelimiters bool, enclosers []string, lineFormat string, ignore string) error {
if utf8.RuneCountInString(delimiters) == 0 { if utf8.RuneCountInString(delimiters) == 0 {
return fmt.Errorf("delimiter shoud be at least one character") return fmt.Errorf("delimiter shoud be at least one character")
} }
p.delimiters = delimiters p.delimiters = delimiters
p.mergeDelimiters = mergeDelimiters p.mergeDelimiters = mergeDelimiters
for _, encloser := range enclosers { for _, encloser := range enclosers {
if utf8.RuneCountInString(encloser) != 2 { if utf8.RuneCountInString(encloser) != 2 {
return fmt.Errorf("encolser should have to characters") return fmt.Errorf("encolser should have to characters")
} }
} }
p.enclosers = enclosers p.enclosers = enclosers
p.ignore = ignore
// line format is in the form of: field1 field2 ignore ... // line format is in the form of: field1 field2 ignore ...
// if field name is ignore, it is parsed but not retained // if field name is ignore, it is parsed but not retained
p.fields = strings.Split(lineFormat, " ") fields := strings.Split(lineFormat, " ")
p.fields = make([]string, 0)
for _, f := range fields {
if len(f) > 0 {
p.fields = append(p.fields, f)
}
}
p.maxFieldIndex = len(p.fields) - 1
return nil return nil
} }
type parserState struct { type ParserState struct {
delimiter bool inField bool // reading field value
enclosed bool delimiter bool // last char was a delimiter
encloserEnd rune escape bool // the previaus char was a escaper \\
enclosed bool // the current field is enclosed
enclosedMode bool // the current algo is for enclosed string, do not search delimiter
encloserStart rune // the current enclosed field opener char
encloserEnd rune // the current enclosed field closer char
} }
/*
* si pas dans la lecture d'un champ
*/
func (p *CsvParser) Parse(line string) (map[string]string, error) { func (p *CsvParser) Parse(line string) (map[string]string, error) {
currentFieldIndex := 0 currentFieldIndex := 0
valueStart := -1 valueStart := -1
escape := false //valueEnd := -1
enclosed := false
state := ParserState{inField: false, delimiter: true, escape: false, enclosed: false, enclosedMode: false, encloserStart: '?', encloserEnd: '?'}
ret := make(map[string]string) ret := make(map[string]string)
indexMax := len(line) - 1
maxFieldIndex := len(p.fields) - 1 maxIndex := len(line) - 1
delimiters := p.delimiters
delimiter := false
for index, r := range line { for index, r := range line {
if r == '\\' { if r == '\\' {
// Check if EOL before continue state.escape = true
escape = true
continue continue
} }
if escape { // previous rune was \\: ignore the current rune
escape = false if state.escape {
state.escape = false
continue continue
} }
for _, d := range delimiters { // reading en enclosed field: watch for end of this field (encloserEnd)
if r == d { if state.enclosedMode {
delimiter = true // searching for end of encloser
break if r == state.encloserEnd {
} state.enclosedMode = false
}
if index < maxIndex {
continue
}
} }
if delimiter && valueStart > -1 { isDelimiter := false
if p.fields[currentFieldIndex] != "ignore" { for _, d := range p.delimiters {
ret[p.fields[currentFieldIndex]] = line[valueStart:index] if r == d {
} isDelimiter = true
currentFieldIndex++
}
if delimiter {
if p.fields[currentFieldIndex] != "ignore" {
ret[p.fields[currentFieldIndex]] = line[valueStart:index]
}
currentFieldIndex++
if currentFieldIndex > maxFieldIndex {
break break
} }
valueStart = index + 1 }
if enclosed {
enclosed = false // previous rune was a delimiter and mergeDelimiters is true, skip
delimiters = p.delimiters if isDelimiter && state.delimiter && p.mergeDelimiters {
// Omit next delimiter
escape = true
}
continue continue
} }
if index >= indexMax { // current rune is a delimiter, a value is present and the next char is the next value
if p.fields[currentFieldIndex] != "ignore" { if isDelimiter || index == maxIndex {
ret[p.fields[currentFieldIndex]] = line[valueStart:] state.delimiter = true
field := line[valueStart:index]
if state.enclosed {
field = strings.TrimRight(strings.TrimLeft(field, string(state.encloserStart)), string(state.encloserEnd))
} }
if currentFieldIndex <= p.maxFieldIndex && p.fields[currentFieldIndex] != p.ignore {
ret[p.fields[currentFieldIndex]] = field
}
currentFieldIndex++
state = ParserState{inField: false, delimiter: true, escape: false, enclosed: false, enclosedMode: false, encloserStart: '?', encloserEnd: '?'}
continue continue
} }
if state.delimiter {
for _, encloser := range p.enclosers { valueStart = index
runes := []rune(encloser) for _, e := range p.enclosers {
if r == runes[0] { runes := []rune(e)
// opening encloser if r == runes[0] {
enclosed = true state.enclosed = true
delimiters = string(runes[1]) state.enclosedMode = true
valueStart++ state.encloserStart = runes[0]
break state.encloserEnd = runes[1]
}
state.delimiter = false
} }
} }
} }

104
csvparser_test.go
View File

@ -7,8 +7,66 @@ import (
func TestCorrectLines(t *testing.T) { func TestCorrectLines(t *testing.T) {
var csvParser CsvParser var csvParser CsvParser
csvParser.Initialize(" ", []string{"\"\"", "[]"}, "firstname lastname complete_name")
line := "" line := ""
/*
csvParser.Initialize(" ", false, []string{"\"\"", "[]"}, "firstname lastname complete_name")
fmt.Println("parsing:", line)
event, err := csvParser.Parse(line)
if err != nil {
t.Fatalf("Parsing of empty line failed %v %v", err, event)
}
fmt.Println("event is", event)
line = "John"
fmt.Println("parsing:", line)
event, err = csvParser.Parse(line)
if err != nil {
t.Fatalf("Parsing of empty line failed %v %v", err, event)
}
fmt.Println("event is", event)
line = "John Doe"
fmt.Println("parsing:", line)
event, err = csvParser.Parse(line)
if err != nil {
t.Fatalf("Parsing of empty line failed %v %v", err, event)
}
fmt.Println("event is", event)
line = "John \"John Doe\" Doe"
fmt.Println("parsing:", line)
event, err = csvParser.Parse(line)
if err != nil {
t.Fatalf("Parsing of empty line failed %v %v", err, event)
}
fmt.Println("event is", event)
line = "John Doe \"John Doe\""
fmt.Println("parsing:", line)
event, err = csvParser.Parse(line)
if err != nil {
t.Fatalf("Parsing of empty line failed %v %v", err, event)
}
fmt.Println("event is", event)
line = "John Doe \"John Doe\"\\\\"
fmt.Println("parsing:", line)
event, err = csvParser.Parse(line)
if err != nil {
t.Fatalf("Parsing of empty line failed %v %v", err, event)
}
fmt.Println("event is", event)
line = "John Doe I don't know him"
fmt.Println("parsing:", line)
event, err = csvParser.Parse(line)
if err != nil {
t.Fatalf("Parsing of empty line failed %v %v", err, event)
}
fmt.Println("event is", event)
*/
csvParser.Initialize(" \t", true, []string{"\"\"", "[]"}, "ignore ignore ignore proxy ignore domain clientip ignore ignore apache-date request status bytes duration referer user-agent", "ignore")
line = "Aug 7 00:00:00 proxy-4 haproxy[17429]: www.yvelines.gouv.fr 66.249.64.10 - - [06/Aug/2024:23:59:59 +0200] \"GET /content/download/19274/117923/file/SE_EAU_20190325_LesJardines_78201900027_LetNotifCompletude+recepisse.pdf HTTP/1.1\" 301 1414 240 \"\" \"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.6478.182 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\" \"-\" \"GB\" \"15169\""
fmt.Println("parsing:", line) fmt.Println("parsing:", line)
event, err := csvParser.Parse(line) event, err := csvParser.Parse(line)
if err != nil { if err != nil {
@ -16,7 +74,7 @@ func TestCorrectLines(t *testing.T) {
} }
fmt.Println("event is", event) fmt.Println("event is", event)
line = "John" line = "Aug 7 00:00:00 proxy-4 haproxy[17429]: www.yvelines.gouv.fr 66.249.64.10 - - [06/Aug/2024:23:59:59 +0200] \"GET /content/download/19274/117923/file/SE_EAU_20190325_LesJardines_78201900027_LetNotifCompletude+recepisse.pdf HTTP/1.1\" 301 1414 240 \"\" \"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.6478.182 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\" \"-\" \"GB\" 15169"
fmt.Println("parsing:", line) fmt.Println("parsing:", line)
event, err = csvParser.Parse(line) event, err = csvParser.Parse(line)
if err != nil { if err != nil {
@ -24,47 +82,7 @@ func TestCorrectLines(t *testing.T) {
} }
fmt.Println("event is", event) fmt.Println("event is", event)
line = "John Doe" line = "Aug 7 00:00:00 proxy-4 haproxy[17429]: www.yvelines.gouv.fr 66.249.64.10 - - [06/Aug/2024:23:59:59 +0200] \"GET /content/download/19274/117923/file/SE_EAU_20190325_LesJardines_78201900027_LetNotifCompletude+recepisse.pdf HTTP/1.1\" 301 1414 240 \"\" \"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.6478.182 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\" \"-\" \"GB\" \"15169"
fmt.Println("parsing:", line)
event, err = csvParser.Parse(line)
if err != nil {
t.Fatalf("Parsing of empty line failed %v %v", err, event)
}
fmt.Println("event is", event)
line = "John \"John Doe\" Doe"
fmt.Println("parsing:", line)
event, err = csvParser.Parse(line)
if err != nil {
t.Fatalf("Parsing of empty line failed %v %v", err, event)
}
fmt.Println("event is", event)
line = "John Doe \"John Doe\""
fmt.Println("parsing:", line)
event, err = csvParser.Parse(line)
if err != nil {
t.Fatalf("Parsing of empty line failed %v %v", err, event)
}
fmt.Println("event is", event)
line = "John Doe \"John Doe\"\\\\"
fmt.Println("parsing:", line)
event, err = csvParser.Parse(line)
if err != nil {
t.Fatalf("Parsing of empty line failed %v %v", err, event)
}
fmt.Println("event is", event)
line = "John Doe I don't know him"
fmt.Println("parsing:", line)
event, err = csvParser.Parse(line)
if err != nil {
t.Fatalf("Parsing of empty line failed %v %v", err, event)
}
csvParser.Initialize(" \t", []string{"\"\"", "[]"}, "ignore ignore ignore proxy ignore domain clientip ignore ignore apache-date request status bytes duration referer user-agent")
fmt.Println("event is", event)
line = "Aug 7 00:00:00 proxy-4 haproxy[17429]: www.yvelines.gouv.fr 66.249.64.10 - - [06/Aug/2024:23:59:59 +0200] \"GET /content/download/19274/117923/file/SE_EAU_20190325_LesJardines_78201900027_LetNotifCompletude+recepisse.pdf HTTP/1.1\" 301 1414 240 \"\" \"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.6478.182 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\" \"-\" \"GB\" \"15169\""
fmt.Println("parsing:", line) fmt.Println("parsing:", line)
event, err = csvParser.Parse(line) event, err = csvParser.Parse(line)
if err != nil { if err != nil {