| // Copyright 2019 The SwiftShader Authors. All Rights Reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| // export_to_sheets updates a Google sheets document with the latest test |
| // results |
| package main |
| |
| import ( |
| "bufio" |
| "bytes" |
| "context" |
| "encoding/json" |
| "flag" |
| "fmt" |
| "io/ioutil" |
| "log" |
| "net/http" |
| "os" |
| "path/filepath" |
| "strings" |
| |
| "./cause" |
| "./consts" |
| "./git" |
| "./testlist" |
| |
| "golang.org/x/oauth2" |
| "golang.org/x/oauth2/google" |
| "google.golang.org/api/sheets/v4" |
| ) |
| |
| var ( |
| authdir = flag.String("authdir", "~/.regres-auth", "directory to hold credentials.json and generated token") |
| projectPath = flag.String("projpath", ".", "project path") |
| testListPath = flag.String("testlist", "tests/regres/full-tests.json", "project relative path to the test list .json file") |
| spreadsheetID = flag.String("spreadsheet", "1RCxbqtKNDG9rVMe_xHMapMBgzOCp24mumab73SbHtfw", "identifier of the spreadsheet to update") |
| ) |
| |
| const ( |
| columnGitHash = "GIT_HASH" |
| columnGitDate = "GIT_DATE" |
| ) |
| |
| func main() { |
| flag.Parse() |
| |
| if err := run(); err != nil { |
| log.Fatalln(err) |
| } |
| } |
| |
| func run() error { |
| // Load the full test list. We use this to find the test file names. |
| lists, err := testlist.Load(".", *testListPath) |
| if err != nil { |
| return cause.Wrap(err, "Unable to load test list") |
| } |
| |
| // Load the creditials used for editing the Google Sheets spreadsheet. |
| srv, err := createSheetsService(*authdir) |
| if err != nil { |
| return cause.Wrap(err, "Unable to authenticate") |
| } |
| |
| // Ensure that there is a sheet for each of the test lists. |
| if err := createTestListSheets(srv, lists); err != nil { |
| return cause.Wrap(err, "Unable to create sheets") |
| } |
| |
| spreadsheet, err := srv.Spreadsheets.Get(*spreadsheetID).Do() |
| if err != nil { |
| return cause.Wrap(err, "Unable to get spreadsheet") |
| } |
| |
| req := sheets.BatchUpdateValuesRequest{ |
| ValueInputOption: "RAW", |
| } |
| |
| testListDir := filepath.Dir(filepath.Join(*projectPath, *testListPath)) |
| changes, err := git.Log(testListDir, 100) |
| if err != nil { |
| return cause.Wrap(err, "Couldn't get git changes for '%v'", testListDir) |
| } |
| |
| for _, group := range lists { |
| sheetName := group.Name |
| fmt.Println("Processing sheet", sheetName) |
| sheet := getSheet(spreadsheet, sheetName) |
| if sheet == nil { |
| return cause.Wrap(err, "Sheet '%v' not found", sheetName) |
| } |
| |
| columnHeaders, err := fetchRow(srv, spreadsheet, sheet, 0) |
| if err != nil { |
| return cause.Wrap(err, "Couldn't get sheet '%v' column headers", sheetName) |
| } |
| |
| columnIndices := listToMap(columnHeaders) |
| |
| hashColumnIndex, found := columnIndices[columnGitHash] |
| if !found { |
| return cause.Wrap(err, "Couldn't find sheet '%v' column header '%v'", sheetName, columnGitHash) |
| } |
| |
| hashValues, err := fetchColumn(srv, spreadsheet, sheet, hashColumnIndex) |
| if err != nil { |
| return cause.Wrap(err, "Couldn't get sheet '%v' column headers", sheetName) |
| } |
| hashValues = hashValues[1:] // Skip header |
| |
| hashIndices := listToMap(hashValues) |
| rowValues := map[string]interface{}{} |
| |
| rowInsertionPoint := 1 + len(hashValues) |
| |
| for i := len(changes) - 1; i >= 0; i-- { |
| change := changes[i] |
| if !strings.HasPrefix(change.Subject, consts.TestListUpdateCommitSubjectPrefix) { |
| continue |
| } |
| |
| hash := change.Hash.String() |
| if _, found := hashIndices[hash]; found { |
| continue // Already in the sheet |
| } |
| |
| rowValues[columnGitHash] = change.Hash.String() |
| rowValues[columnGitDate] = change.Date.Format("2006-01-02") |
| |
| path := filepath.Join(*projectPath, group.File) |
| hasData := false |
| for _, status := range testlist.Statuses { |
| path := testlist.FilePathWithStatus(path, status) |
| data, err := git.Show(path, hash) |
| if err != nil { |
| continue |
| } |
| lines, err := countLines(data) |
| if err != nil { |
| return cause.Wrap(err, "Couldn't count lines in file '%s'", path) |
| } |
| |
| rowValues[string(status)] = lines |
| hasData = true |
| } |
| |
| if !hasData { |
| continue |
| } |
| |
| data, err := mapToList(columnIndices, rowValues) |
| if err != nil { |
| return cause.Wrap(err, "Couldn't map row values to column for sheet %v. Column headers: [%+v]", sheetName, columnHeaders) |
| } |
| |
| req.Data = append(req.Data, &sheets.ValueRange{ |
| Range: rowRange(rowInsertionPoint, sheet), |
| Values: [][]interface{}{data}, |
| }) |
| rowInsertionPoint++ |
| |
| fmt.Printf("Adding test data at %v to %v\n", hash[:8], sheetName) |
| } |
| } |
| |
| if _, err := srv.Spreadsheets.Values.BatchUpdate(*spreadsheetID, &req).Do(); err != nil { |
| return cause.Wrap(err, "Values BatchUpdate failed") |
| } |
| |
| return nil |
| } |
| |
| // listToMap returns the list l as a map where the key is the stringification |
| // of the element, and the value is the element index. |
| func listToMap(l []interface{}) map[string]int { |
| out := map[string]int{} |
| for i, v := range l { |
| out[fmt.Sprint(v)] = i |
| } |
| return out |
| } |
| |
| // mapToList transforms the two maps into a single slice of values. |
| // indices is a map of identifier to output slice element index. |
| // values is a map of identifier to value. |
| func mapToList(indices map[string]int, values map[string]interface{}) ([]interface{}, error) { |
| out := []interface{}{} |
| for name, value := range values { |
| index, ok := indices[name] |
| if !ok { |
| return nil, fmt.Errorf("No index for '%v'", name) |
| } |
| for len(out) <= index { |
| out = append(out, nil) |
| } |
| out[index] = value |
| } |
| return out, nil |
| } |
| |
| // countLines returns the number of new lines in the byte slice data. |
| func countLines(data []byte) (int, error) { |
| scanner := bufio.NewScanner(bytes.NewReader(data)) |
| lines := 0 |
| for scanner.Scan() { |
| lines++ |
| } |
| return lines, nil |
| } |
| |
| // getSheet returns the sheet with the given title name, or nil if the sheet |
| // cannot be found. |
| func getSheet(spreadsheet *sheets.Spreadsheet, name string) *sheets.Sheet { |
| for _, sheet := range spreadsheet.Sheets { |
| if sheet.Properties.Title == name { |
| return sheet |
| } |
| } |
| return nil |
| } |
| |
| // rowRange returns a sheets range ("name!Ai:i") for the entire row with the |
| // given index. |
| func rowRange(index int, sheet *sheets.Sheet) string { |
| return fmt.Sprintf("%v!A%v:%v", sheet.Properties.Title, index+1, index+1) |
| } |
| |
| // columnRange returns a sheets range ("name!i1:i") for the entire column with |
| // the given index. |
| func columnRange(index int, sheet *sheets.Sheet) string { |
| col := 'A' + index |
| if index > 25 { |
| panic("UNIMPLEMENTED") |
| } |
| return fmt.Sprintf("%v!%c1:%c", sheet.Properties.Title, col, col) |
| } |
| |
| // fetchRow returns all the values in the given sheet's row. |
| func fetchRow(srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, row int) ([]interface{}, error) { |
| rng := rowRange(row, sheet) |
| data, err := srv.Spreadsheets.Values.Get(spreadsheet.SpreadsheetId, rng).Do() |
| if err != nil { |
| return nil, cause.Wrap(err, "Couldn't fetch %v", rng) |
| } |
| return data.Values[0], nil |
| } |
| |
| // fetchColumn returns all the values in the given sheet's column. |
| func fetchColumn(srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, row int) ([]interface{}, error) { |
| rng := columnRange(row, sheet) |
| data, err := srv.Spreadsheets.Values.Get(spreadsheet.SpreadsheetId, rng).Do() |
| if err != nil { |
| return nil, cause.Wrap(err, "Couldn't fetch %v", rng) |
| } |
| out := make([]interface{}, len(data.Values)) |
| for i, l := range data.Values { |
| if len(l) > 0 { |
| out[i] = l[0] |
| } |
| } |
| return out, nil |
| } |
| |
| // insertRows inserts blank rows into the given sheet. |
| func insertRows(srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, aboveRow, count int) error { |
| req := sheets.BatchUpdateSpreadsheetRequest{ |
| Requests: []*sheets.Request{{ |
| InsertRange: &sheets.InsertRangeRequest{ |
| Range: &sheets.GridRange{ |
| SheetId: sheet.Properties.SheetId, |
| StartRowIndex: int64(aboveRow), |
| EndRowIndex: int64(aboveRow + count), |
| }, |
| ShiftDimension: "ROWS", |
| }}, |
| }, |
| } |
| if _, err := srv.Spreadsheets.BatchUpdate(*spreadsheetID, &req).Do(); err != nil { |
| return cause.Wrap(err, "Values BatchUpdate failed") |
| } |
| return nil |
| } |
| |
| // createTestListSheets adds a new sheet for each of the test lists, if they |
| // do not already exist. These new sheets are populated with column headers. |
| func createTestListSheets(srv *sheets.Service, testlists testlist.Lists) error { |
| spreadsheet, err := srv.Spreadsheets.Get(*spreadsheetID).Do() |
| if err != nil { |
| return cause.Wrap(err, "Unable to get spreadsheet") |
| } |
| |
| spreadsheetReq := sheets.BatchUpdateSpreadsheetRequest{} |
| updateReq := sheets.BatchUpdateValuesRequest{ValueInputOption: "RAW"} |
| headers := []interface{}{columnGitHash, columnGitDate} |
| for _, s := range testlist.Statuses { |
| headers = append(headers, string(s)) |
| } |
| |
| for _, group := range testlists { |
| name := group.Name |
| if getSheet(spreadsheet, name) == nil { |
| spreadsheetReq.Requests = append(spreadsheetReq.Requests, &sheets.Request{ |
| AddSheet: &sheets.AddSheetRequest{ |
| Properties: &sheets.SheetProperties{ |
| Title: name, |
| }, |
| }, |
| }) |
| updateReq.Data = append(updateReq.Data, |
| &sheets.ValueRange{ |
| Range: name + "!A1:Z", |
| Values: [][]interface{}{headers}, |
| }, |
| ) |
| } |
| } |
| |
| if len(spreadsheetReq.Requests) > 0 { |
| if _, err := srv.Spreadsheets.BatchUpdate(*spreadsheetID, &spreadsheetReq).Do(); err != nil { |
| return cause.Wrap(err, "Spreadsheets BatchUpdate failed") |
| } |
| } |
| if len(updateReq.Data) > 0 { |
| if _, err := srv.Spreadsheets.Values.BatchUpdate(*spreadsheetID, &updateReq).Do(); err != nil { |
| return cause.Wrap(err, "Values BatchUpdate failed") |
| } |
| } |
| |
| return nil |
| } |
| |
| // createSheetsService creates a new Google Sheets service using the credentials |
| // in the credentials.json file. |
| func createSheetsService(authdir string) (*sheets.Service, error) { |
| authdir = os.ExpandEnv(authdir) |
| if home, err := os.UserHomeDir(); err == nil { |
| authdir = strings.ReplaceAll(authdir, "~", home) |
| } |
| |
| os.MkdirAll(authdir, 0777) |
| |
| credentialsPath := filepath.Join(authdir, "credentials.json") |
| b, err := ioutil.ReadFile(credentialsPath) |
| if err != nil { |
| return nil, cause.Wrap(err, "Unable to read client secret file '%v'\n"+ |
| "Obtain this file from: https://console.developers.google.com/apis/credentials", credentialsPath) |
| } |
| |
| config, err := google.ConfigFromJSON(b, "https://www.googleapis.com/auth/spreadsheets") |
| if err != nil { |
| return nil, cause.Wrap(err, "Unable to parse client secret file to config") |
| } |
| |
| client, err := getClient(authdir, config) |
| if err != nil { |
| return nil, cause.Wrap(err, "Unable obtain client") |
| } |
| |
| srv, err := sheets.New(client) |
| if err != nil { |
| return nil, cause.Wrap(err, "Unable to retrieve Sheets client") |
| } |
| return srv, nil |
| } |
| |
| // Retrieve a token, saves the token, then returns the generated client. |
| func getClient(authdir string, config *oauth2.Config) (*http.Client, error) { |
| // The file token.json stores the user's access and refresh tokens, and is |
| // created automatically when the authorization flow completes for the first |
| // time. |
| tokFile := filepath.Join(authdir, "token.json") |
| tok, err := tokenFromFile(tokFile) |
| if err != nil { |
| tok, err = getTokenFromWeb(config) |
| if err != nil { |
| return nil, cause.Wrap(err, "Unable to get token from web") |
| } |
| if err := saveToken(tokFile, tok); err != nil { |
| log.Println("Warning: failed to write token: %v", err) |
| } |
| } |
| return config.Client(context.Background(), tok), nil |
| } |
| |
| // Request a token from the web, then returns the retrieved token. |
| func getTokenFromWeb(config *oauth2.Config) (*oauth2.Token, error) { |
| authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline) |
| fmt.Printf("Go to the following link in your browser then type the "+ |
| "authorization code: \n%v\n", authURL) |
| |
| var authCode string |
| if _, err := fmt.Scan(&authCode); err != nil { |
| return nil, cause.Wrap(err, "Unable to read authorization code") |
| } |
| |
| tok, err := config.Exchange(context.TODO(), authCode) |
| if err != nil { |
| return nil, cause.Wrap(err, "Unable to retrieve token from web") |
| } |
| return tok, nil |
| } |
| |
| // Retrieves a token from a local file. |
| func tokenFromFile(path string) (*oauth2.Token, error) { |
| f, err := os.Open(path) |
| if err != nil { |
| return nil, err |
| } |
| defer f.Close() |
| tok := &oauth2.Token{} |
| err = json.NewDecoder(f).Decode(tok) |
| return tok, err |
| } |
| |
| // Saves a token to a file path. |
| func saveToken(path string, token *oauth2.Token) error { |
| fmt.Printf("Saving credential file to: %s\n", path) |
| f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) |
| if err != nil { |
| return cause.Wrap(err, "Unable to cache oauth token") |
| } |
| defer f.Close() |
| json.NewEncoder(f).Encode(token) |
| return nil |
| } |