Algo ok, mais peut-être à optimiser ?

csvparser.go

@@ -9,8 +9,10 @@ import (
 type CsvParser struct {
 	enclosers       []string
 	delimiters      string
-  mergeDelimiters string
+	mergeDelimiters bool
 	fields          []string
+	ignore          string
+	maxFieldIndex   int
 }
 
 /*
@@ -19,7 +21,8 @@ type CsvParser struct {
 * enclosers: array of string, each string contains a enclorser tuple: "", [], {}...
 * lineFormat: format of the line (each fieldname or ignore)
  */
-func (p *CsvParser) Initialize(delimiters string, mergeDelimiters bool, enclosers []string, lineFormat string) error {
+func (p *CsvParser) Initialize(delimiters string, mergeDelimiters bool, enclosers []string, lineFormat string, ignore string) error {
+
 	if utf8.RuneCountInString(delimiters) == 0 {
 		return fmt.Errorf("delimiter shoud be at least one character")
 	}
@@ -31,85 +34,103 @@ func (p *CsvParser) Initialize(delimiters string, mergeDelimiters bool, encloser
 		}
 	}
 	p.enclosers = enclosers
+
+	p.ignore = ignore
 	// line format is in the form of: field1 field2 ignore ...
 	// if field name is ignore, it is parsed but not retained
-	p.fields = strings.Split(lineFormat, " ")
+	fields := strings.Split(lineFormat, " ")
 
+	p.fields = make([]string, 0)
+	for _, f := range fields {
+		if len(f) > 0 {
+			p.fields = append(p.fields, f)
+		}
+	}
+	p.maxFieldIndex = len(p.fields) - 1
 	return nil
 }
 
-type parserState struct {
+type ParserState struct {
-  delimiter bool
+	inField       bool // reading field value
-  enclosed bool
+	delimiter     bool // last char was a delimiter
-  encloserEnd rune
+	escape        bool // the previaus char was a escaper \\
+	enclosed      bool // the current field is enclosed
+	enclosedMode  bool // the current algo is for enclosed string, do not search delimiter
+	encloserStart rune // the current enclosed field opener char
+	encloserEnd   rune // the current enclosed field closer char
 }
 
+/*
+* si pas dans la lecture d'un champ
+ */
 func (p *CsvParser) Parse(line string) (map[string]string, error) {
 	currentFieldIndex := 0
 	valueStart := -1
-	escape := false
+	//valueEnd := -1
-	enclosed := false
+
+	state := ParserState{inField: false, delimiter: true, escape: false, enclosed: false, enclosedMode: false, encloserStart: '?', encloserEnd: '?'}
 
 	ret := make(map[string]string)
-	indexMax := len(line) - 1
+
-	maxFieldIndex := len(p.fields) - 1
+	maxIndex := len(line) - 1
-	delimiters := p.delimiters
-  delimiter := false
 	for index, r := range line {
 		if r == '\\' {
-			// Check if EOL before continue
+			state.escape = true
-			escape = true
 			continue
 		}
-		if escape {
+		// previous rune was \\: ignore the current rune
-			escape = false
+		if state.escape {
+			state.escape = false
 			continue
 		}
-		for _, d := range delimiters {
+		// reading en enclosed field: watch for end of this field (encloserEnd)
+		if state.enclosedMode {
+			// searching for end of encloser
+			if r == state.encloserEnd {
+				state.enclosedMode = false
+			}
+			if index < maxIndex {
+				continue
+			}
+		}
+		isDelimiter := false
+		for _, d := range p.delimiters {
 			if r == d {
-        delimiter = true
+				isDelimiter = true
 				break
 			}
 		}
-    if delimiter && valueStart > -1 {
+
-			if p.fields[currentFieldIndex] != "ignore" {
+		// previous rune was a delimiter and mergeDelimiters is true, skip
-				ret[p.fields[currentFieldIndex]] = line[valueStart:index]
+		if isDelimiter && state.delimiter && p.mergeDelimiters {
-			}
-			currentFieldIndex++
-    }
-		if delimiter {
-			if p.fields[currentFieldIndex] != "ignore" {
-				ret[p.fields[currentFieldIndex]] = line[valueStart:index]
-			}
-			currentFieldIndex++
-			if currentFieldIndex > maxFieldIndex {
-				break
-			}
-			valueStart = index + 1
-			if enclosed {
-				enclosed = false
-				delimiters = p.delimiters
-				// Omit next delimiter
-				escape = true
-			}
 			continue
 		}
 
-		if index >= indexMax {
+		// current rune is a delimiter, a value is present and the next char is the next value
-			if p.fields[currentFieldIndex] != "ignore" {
+		if isDelimiter || index == maxIndex {
-				ret[p.fields[currentFieldIndex]] = line[valueStart:]
+			state.delimiter = true
+			field := line[valueStart:index]
+			if state.enclosed {
+				field = strings.TrimRight(strings.TrimLeft(field, string(state.encloserStart)), string(state.encloserEnd))
 			}
+			if currentFieldIndex <= p.maxFieldIndex && p.fields[currentFieldIndex] != p.ignore {
+				ret[p.fields[currentFieldIndex]] = field
+			}
+			currentFieldIndex++
+			state = ParserState{inField: false, delimiter: true, escape: false, enclosed: false, enclosedMode: false, encloserStart: '?', encloserEnd: '?'}
 			continue
 		}
-
+		if state.delimiter {
-		for _, encloser := range p.enclosers {
+			valueStart = index
-			runes := []rune(encloser)
+			for _, e := range p.enclosers {
+				runes := []rune(e)
 				if r == runes[0] {
-				// opening encloser
+					state.enclosed = true
-				enclosed = true
+					state.enclosedMode = true
-				delimiters = string(runes[1])
+					state.encloserStart = runes[0]
-				valueStart++
+					state.encloserEnd = runes[1]
-				break
+				}
+				state.delimiter = false
 			}
 		}
 	}

csvparser_test.go

@@ -7,8 +7,9 @@ import (
 
 func TestCorrectLines(t *testing.T) {
 	var csvParser CsvParser
-	csvParser.Initialize(" ", []string{"\"\"", "[]"}, "firstname lastname complete_name")
 	line := ""
+	/*
+		csvParser.Initialize(" ", false, []string{"\"\"", "[]"}, "firstname lastname complete_name")
 		fmt.Println("parsing:", line)
 		event, err := csvParser.Parse(line)
 		if err != nil {
@@ -62,10 +63,27 @@ func TestCorrectLines(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Parsing of empty line failed %v %v", err, event)
 		}
-	csvParser.Initialize(" \t", []string{"\"\"", "[]"}, "ignore ignore ignore proxy ignore domain clientip ignore ignore apache-date request status bytes duration referer user-agent")
 		fmt.Println("event is", event)
+	*/
+	csvParser.Initialize(" \t", true, []string{"\"\"", "[]"}, "ignore ignore ignore proxy ignore domain clientip ignore ignore apache-date request status bytes duration referer user-agent", "ignore")
 	line = "Aug  7 00:00:00 proxy-4 haproxy[17429]: www.yvelines.gouv.fr 66.249.64.10 - - [06/Aug/2024:23:59:59 +0200] \"GET /content/download/19274/117923/file/SE_EAU_20190325_LesJardines_78201900027_LetNotifCompletude+recepisse.pdf HTTP/1.1\" 301 1414 240 \"\" \"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.6478.182 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\" \"-\" \"GB\" \"15169\""
 	fmt.Println("parsing:", line)
+	event, err := csvParser.Parse(line)
+	if err != nil {
+		t.Fatalf("Parsing of empty line failed %v %v", err, event)
+	}
+	fmt.Println("event is", event)
+
+	line = "Aug  7 00:00:00 proxy-4 haproxy[17429]: www.yvelines.gouv.fr 66.249.64.10 - - [06/Aug/2024:23:59:59 +0200] \"GET /content/download/19274/117923/file/SE_EAU_20190325_LesJardines_78201900027_LetNotifCompletude+recepisse.pdf HTTP/1.1\" 301 1414 240 \"\" \"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.6478.182 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\" \"-\" \"GB\" 15169"
+	fmt.Println("parsing:", line)
+	event, err = csvParser.Parse(line)
+	if err != nil {
+		t.Fatalf("Parsing of empty line failed %v %v", err, event)
+	}
+	fmt.Println("event is", event)
+
+	line = "Aug  7 00:00:00 proxy-4 haproxy[17429]: www.yvelines.gouv.fr 66.249.64.10 - - [06/Aug/2024:23:59:59 +0200] \"GET /content/download/19274/117923/file/SE_EAU_20190325_LesJardines_78201900027_LetNotifCompletude+recepisse.pdf HTTP/1.1\" 301 1414 240 \"\" \"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.6478.182 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\" \"-\" \"GB\" \"15169"
+	fmt.Println("parsing:", line)
 	event, err = csvParser.Parse(line)
 	if err != nil {
 		t.Fatalf("Parsing of empty line failed %v %v", err, event)