Skip to content

Commit c23ba65

Browse files
mpictormpictor
committed
appveyor log - use JSON-like console data, massaged for parsability
1 parent 274de2a commit c23ba65

File tree

1 file changed

+156
-47
lines changed

1 file changed

+156
-47
lines changed

‎misc/summarize-appveyor-log.go‎

Lines changed: 156 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
//summarize MSVC errors from an appveyor log
22
// compile with 'go build summarize-appveyor-log.go'
3+
// takes 0 or 1 args; with 0, gets log from latest
4+
// build. with 1, uses that file as raw json-like log
35
package main
46

57
import (
@@ -17,22 +19,29 @@ import (
1719
const (
1820
headerKey = "Authorization"
1921
headerVal = "Bearer %s"
20-
projUrl = "https://ci.appveyor.com/api/projects/mpictor/stepcode"
22+
projUrl = "https://ci.appveyor.com/api/projects/mpictor/stepcode"
2123
//"https://ci.appveyor.com/api/buildjobs/2rjxdv1rnb8jcg8y/log"
22-
logUrl = "https://ci.appveyor.com/api/buildjobs/%s/log"
24+
logUrl = "https://ci.appveyor.com/api/buildjobs/%s/log"
25+
consoleUrl = "https://ci.appveyor.com/api/buildjobs/%s/console"
2326
)
2427

25-
//uses stdin and stdout
2628
func main() {
27-
rawlog, build, err := getLog()
29+
var rawlog io.ReadCloser
30+
var build string
31+
var err error
32+
if len(os.Args) == 2 {
33+
rawlog, build, err = processArgv()
34+
} else {
35+
rawlog, build, err = getLog()
36+
}
2837
if err != nil {
2938
fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
3039
return
3140
}
3241
defer rawlog.Close()
33-
log := unwrap(rawlog)
42+
log := decodeConsole(rawlog)
3443
warns, errs := countMessages(log)
35-
fi, err := os.Create(fmt.Sprintf("appveyor-%d.smy", build))
44+
fi, err := os.Create(fmt.Sprintf("appveyor-%s.smy", build))
3645
if err != nil {
3746
fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
3847
return
@@ -46,20 +55,19 @@ func main() {
4655

4756
/* categorizes warnings and errors based upon the MSVC message number (i.e. C4244)
4857
* the regex will match lines like
49-
[ 00:03:42] c:\projects\stepcode\src\base\sc_benchmark.h(45): warning C4251: 'benchmark::descr' : class 'std::basic_string<char,std::char_traits<char>,std::allocator<char>>' needs to have dll-interface to be used by clients of class 'benchmark' [C:\projects\STEPcode\build\src\base\base.vcxproj]
58+
c:\projects\stepcode\src\base\sc_benchmark.h(45): warning C4251: 'benchmark::descr' : class 'std::basic_string<char,std::char_traits<char>,std::allocator<char>>' needs to have dll-interface to be used by clients of class 'benchmark' [C:\projects\STEPcode\build\src\base\base.vcxproj]
5059
[00:03:48] C:\projects\STEPcode\src\base\sc_benchmark.cc(61): warning C4244: '=' : conversion from 'SIZE_T' to 'long', possible loss of data [C:\projects\STEPcode\build\src\base\base.vcxproj]*
5160
*/
5261
func countMessages(log []string) (warns, errs map[string][]string) {
5362
warns = make(map[string][]string)
5463
errs = make(map[string][]string)
55-
tstamp := `\[\d\d:\d\d:\d\d\] `
5664
fname := " *(.*)" // $1
5765
fline := `(?:\((\d+)\)| ): ` // $2 - either line number in parenthesis or a space, followed by a colon
5866
msgNr := `([A-Z]+\d+): ` // $3 - C4251, LNK2005, etc
5967
msgTxt := `([^\[]*) ` // $4
6068
tail := `\[[^\[\]]*\]`
61-
warnRe := regexp.MustCompile(tstamp + fname + fline + `warning ` + msgNr + msgTxt + tail)
62-
errRe := regexp.MustCompile(tstamp + fname + fline + `(?:fatal )?error ` + msgNr + msgTxt + tail)
69+
warnRe := regexp.MustCompile(fname + fline + `warning ` + msgNr + msgTxt + tail)
70+
errRe := regexp.MustCompile(fname + fline + `(?:fatal )?error ` + msgNr + msgTxt + tail)
6371
for _, line := range log {
6472
if warnRe.MatchString(line) {
6573
key := warnRe.ReplaceAllString(line, "$3")
@@ -125,68 +133,126 @@ func printMessages(typ string, m map[string][]string, w io.Writer) {
125133
}
126134
}
127135

128-
//
129-
func unwrap(r io.Reader) (log []string) {
130-
startNewLine := true
136+
//structs from http://json2struct.mervine.net/
137+
138+
//{"values":[{"i":0,"t":"Specify a project or solution file. The directory does not contain a project or solution file.\r\n","dt":"00:00:04","bg":12,"fg":15}]}
139+
type AppVeyorConsoleLines struct {
140+
Values []struct {
141+
I int `json:"i"`
142+
Text string `json:"t"`
143+
DateTime string `json:"dt"`
144+
BgColor int `json:"bg"`
145+
FgColor int `json:"fg"`
146+
}
147+
}
148+
type AppVeyorBuild struct {
149+
Build struct {
150+
/*BuildNumber int `json:"buildNumber"`*/
151+
Version string `json:"version"`
152+
Jobs []struct {
153+
JobID string `json:"jobId"`
154+
} `json:"jobs"`
155+
} `json:"build"`
156+
}
157+
158+
func splitAppend(log *[]string, blob string) {
159+
//blob = strings.Replace(blob,"\r\n", "\n",-1)
160+
blob = strings.Replace(blob, "\\", "/", -1)
161+
r := strings.NewReader(blob)
131162
unwrapScanner := bufio.NewScanner(r)
132-
var lineOut string
133163
for unwrapScanner.Scan() {
134-
lastNewline := startNewLine
135-
lineIn := unwrapScanner.Text()
136-
startNewLine = (len(lineIn) < 240) || strings.HasSuffix(lineIn, "vcxproj]")
137-
if !lastNewline {
138-
lineOut += lineIn[11:]
139-
} else {
140-
lineOut = lineIn
141-
}
142-
if startNewLine {
143-
log = append(log, lineOut)
144-
lineOut = ""
145-
}
164+
txt := unwrapScanner.Text()
165+
//fmt.Printf("%s\n", txt)
166+
*log = append(*log, txt)
146167
}
147-
if len(lineOut) > 0 {
148-
log = append(log, lineOut)
168+
}
169+
170+
//calculate length of string without escape chars
171+
// func escapeLen(s string)(l int) {
172+
// //s = strings.Replace(s,"\\\\", "/",-1)
173+
// s = strings.Replace(s,"\\\"", "",-1)
174+
// s = strings.Replace(s,"\r\n", "RN",-1)
175+
// return len(s)
176+
// }
177+
178+
179+
//decode the almost-JSON console data from appveyor
180+
func decodeConsole(r io.Reader) (log []string) {
181+
wrapper := Wrap(r)
182+
dec := json.NewDecoder(wrapper)
183+
var consoleLines AppVeyorConsoleLines
184+
var err error
185+
var txtBlob string
186+
err = dec.Decode(&consoleLines)
187+
if err == io.EOF {
188+
err = nil
149189
}
150-
if err := unwrapScanner.Err(); err != nil {
151-
fmt.Fprintln(os.Stderr, "Error reading appveyor log:", err)
190+
if err == nil {
191+
for _, l := range consoleLines.Values {
192+
txtBlob += l.Text
193+
//el := escapeLen(l.Text)
194+
//something inserts newlines at 229 chars (+\n\r == 231) (found in CMake output)
195+
lenTwoThreeOne := len(l.Text) == 231
196+
if lenTwoThreeOne {
197+
txtBlob = strings.TrimSuffix(txtBlob, "\r\n")
198+
}
199+
//something else starts new log lines at 1024 chars without inserting newlines (found in CTest error output)
200+
if len(l.Text) != 1024 && !lenTwoThreeOne {
201+
//fmt.Printf("sa for l %d, el %d\n", len(l.Text),el)
202+
splitAppend(&log, txtBlob)
203+
txtBlob = ""
204+
}
205+
}
206+
} else {
207+
fmt.Printf("decode err %s\n", err)
208+
}
209+
if len(txtBlob) > 0 {
210+
splitAppend(&log, txtBlob)
152211
}
153212
return
154213
}
155214

156-
//http://json2struct.mervine.net/
157-
type AppVeyorBuild struct {
158-
Build struct {
159-
BuildNumber int `json:"buildNumber"`
160-
Jobs []struct {
161-
JobID string `json:"jobId"`
162-
} `json:"jobs"`
163-
} `json:"build"`
215+
func processArgv() (log io.ReadCloser, build string, err error) {
216+
fname := os.Args[1]
217+
if len(fname) < 14 {
218+
err = fmt.Errorf("Name arg '%s' too short. Run as '%s appveyor-NNN.log'", fname, os.Args[0])
219+
return
220+
}
221+
buildRe := regexp.MustCompile(`appveyor-(.+).log`)
222+
build = buildRe.ReplaceAllString(fname, "$1")
223+
if len(build) == 0 {
224+
err = fmt.Errorf("No build id in %s", fname)
225+
return
226+
}
227+
log, err = os.Open(fname)
228+
return
164229
}
165230

166-
func getLog() (log io.ReadCloser, build int, err error) {
231+
func getLog() (log io.ReadCloser, build string, err error) {
167232
client := &http.Client{}
168233
req, err := http.NewRequest("GET", projUrl, nil)
169234
if err != nil {
170235
return
171236
}
172237
apikey := os.Getenv("APPVEYOR_API_KEY")
173238
//api key isn't necessary for read-only queries on public projects
174-
//if len(apikey) < 1 {
239+
if len(apikey) > 0 {
240+
req.Header.Add(headerKey, fmt.Sprintf(headerVal, apikey))
241+
} //else {
175242
// fmt.Printf("Env var APPVEYOR_API_KEY is not set.")
176243
//}
177-
req.Header.Add(headerKey, fmt.Sprintf(headerVal,apikey))
178244
resp, err := client.Do(req)
179245
if err != nil {
180246
return
181247
}
182248

183-
build, job := decode(resp.Body)
184-
fmt.Printf("build #%d, jobId %s\n", build, job)
185-
resp, err = http.Get(fmt.Sprintf(logUrl, job))
249+
build, job := decodeProjInfo(resp.Body)
250+
fmt.Printf("build #%s, jobId %s\n", build, job)
251+
resp, err = http.Get(fmt.Sprintf(consoleUrl, job))
186252
if err != nil {
187253
return
188254
}
189-
logName := fmt.Sprintf("appveyor-%d.log", build)
255+
logName := fmt.Sprintf("appveyor-%s.log", build)
190256
fi, err := os.Create(logName)
191257
if err != nil {
192258
return
@@ -202,7 +268,7 @@ func getLog() (log io.ReadCloser, build int, err error) {
202268
return
203269
}
204270

205-
func decode(r io.Reader) (num int, job string) {
271+
func decodeProjInfo(r io.Reader) (vers string, job string) {
206272
dec := json.NewDecoder(r)
207273
var av AppVeyorBuild
208274
err := dec.Decode(&av)
@@ -213,9 +279,52 @@ func decode(r io.Reader) (num int, job string) {
213279
if len(av.Build.Jobs) != 1 {
214280
return
215281
}
216-
num = av.Build.BuildNumber
282+
vers = av.Build.Version
217283
job = av.Build.Jobs[0].JobID
218284
return
219285
}
220286

287+
//wrap a reader, modifying content to make the json decoder happy
288+
//only tested with data from appveyor console
289+
type jsonWrapper struct {
290+
source io.Reader
291+
begin bool
292+
end bool
293+
}
294+
295+
func Wrap(r io.Reader) *jsonWrapper {
296+
return &jsonWrapper{
297+
source: r,
298+
begin: true,
299+
}
300+
}
301+
302+
// func nonNeg(n int) (int) {
303+
// if n < 0 {
304+
// return 0
305+
// }
306+
// return n
307+
// }
308+
309+
func (w *jsonWrapper) Read(p []byte) (n int, err error) {
310+
if w.end {
311+
return 0, io.EOF
312+
}
313+
if w.begin {
314+
w.begin = false
315+
n = copy(p, []byte(`{"values":[`))
316+
}
317+
m, err := w.source.Read(p[n:])
318+
n += m
319+
if err == io.EOF {
320+
w.end = true
321+
if n < len(p) {
322+
n = copy(p, []byte(`{"dummy":"data"}]}`))
323+
} else {
324+
err = fmt.Errorf("No room to terminate JSON struct with '}'\n")
325+
}
326+
}
327+
return
328+
}
329+
221330
// kate: indent-width 8; space-indent off; replace-tabs off; replace-tabs-save off; replace-trailing-space-save on; remove-trailing-space on; tab-intent on; tab-width 8; show-tabs off;

0 commit comments

Comments
 (0)