grim/youtrack-import

user maps with bitbucket work. Still needs more testing though

2019-12-31, Gary Kramlich
fab48894ef61
Parents 75076ea0a1ee
Children ac0a0ad87f41
user maps with bitbucket work. Still needs more testing though
11 files changed, 346 insertions(+), 14 deletions(-)
  • +71 -0
    bitbucket/archive.go
  • +8 -2
    bitbucket/cmd.go
  • +23 -5
    bitbucket/converter.go
  • +57 -0
    bitbucket/usersmap.go
  • +1 -1
    build-and-run
  • +1 -1
    youtrack/client.go
  • +5 -0
    youtrack/import.go
  • +11 -5
    youtrack/issue.go
  • +2 -0
    youtrack/project.go
  • +129 -0
    youtrack/user.go
  • +38 -0
    youtrack/xml.go
    • --- a/bitbucket/archive.go Mon Dec 30 20:17:16 2019 -0600
    +++ b/bitbucket/archive.go Tue Dec 31 01:08:27 2019 -0600
    @@ -4,6 +4,8 @@
    "archive/zip"
    "encoding/json"
    "fmt"
    +
    + "hg.sr.ht/~grim/youtrack-import/youtrack"
    )
    func loadArchive(archive string) (*Archive, error) {
    @@ -32,3 +34,72 @@
    return nil, fmt.Errorf("failed to find db-2.0.json in the archive")
    }
    +
    +func (a *Archive) ExtractUsers(usersMap *UsersMap) (map[string]*youtrack.User, error) {
    + // key is the id of the user
    + users := map[string]*youtrack.User{}
    +
    + mapUser := func(a *Author) error {
    + if a.AccountID != "" {
    + bbUser, err := usersMap.Find(a.DisplayName)
    + if err != nil {
    + return err
    + }
    +
    + users[a.AccountID] = bbUser.ToYoutrack()
    + }
    +
    + return nil
    + }
    +
    + // find all users from issues
    + for _, issue := range a.Issues {
    + if err := mapUser(&issue.Assignee); err != nil {
    + return nil, err
    + }
    +
    + if err := mapUser(&issue.Reporter); err != nil {
    + return nil, err
    + }
    +
    + for _, watcher := range issue.Watchers {
    + if err := mapUser(&watcher); err != nil {
    + return nil, err
    + }
    + }
    +
    + for _, voter := range issue.Voters {
    + if err := mapUser(&voter); err != nil {
    + return nil, err
    + }
    + }
    + }
    +
    + // find all users from comments
    + for _, comment := range a.Comments {
    + if err := mapUser(&comment.User); err != nil {
    + return nil, err
    + }
    + }
    +
    + // find all users from attachments
    + for _, attachment := range a.Attachments {
    + if err := mapUser(&attachment.User); err != nil {
    + return nil, err
    + }
    + }
    +
    + // find all users from logs
    + for _, log := range a.Logs {
    + if err := mapUser(&log.User); err != nil {
    + return nil, err
    + }
    + }
    +
    + // extract the user from the default assignee
    + if err := mapUser(&a.Meta.DefaultAssignee); err != nil {
    + return nil, err
    + }
    +
    + return users, nil
    +}
    --- a/bitbucket/cmd.go Mon Dec 30 20:17:16 2019 -0600
    +++ b/bitbucket/cmd.go Tue Dec 31 01:08:27 2019 -0600
    @@ -6,7 +6,8 @@
    )
    type Cmd struct {
    - Archive string `kong:"arg,name='archive',help='The zip file containing the archive'"`
    + Archive string `kong:"arg,name='archive',help='The zip file containing the archive'"`
    + UsersMapFile string `kong:"arg,name="users-map",help='A key=value file mapping emails to display names'"`
    }
    func (c *Cmd) Run(g *globals.Globals) error {
    @@ -15,7 +16,12 @@
    return err
    }
    - project, err := archive.convert()
    + usersMap, err := NewUsersMap(c.UsersMapFile)
    + if err != nil {
    + return err
    + }
    +
    + project, err := archive.convert(usersMap)
    if err != nil {
    return err
    }
    --- a/bitbucket/converter.go Mon Dec 30 20:17:16 2019 -0600
    +++ b/bitbucket/converter.go Tue Dec 31 01:08:27 2019 -0600
    @@ -32,14 +32,19 @@
    }
    )
    -func (a *Archive) convertIssue(bb Issue) *youtrack.Issue {
    +func (a *Archive) convertIssue(bb Issue, userMap map[string]*youtrack.User) *youtrack.Issue {
    + reporter := "guest"
    + if user, found := userMap[bb.Reporter.AccountID]; found {
    + reporter = user.Login
    + }
    +
    yt := &youtrack.Issue{
    Number: bb.ID,
    Summary: bb.Title,
    Description: bb.Content,
    Created: bb.CreatedOn,
    Updated: bb.UpdatedOn,
    - Reporter: bb.Reporter.String(),
    + Reporter: reporter,
    Priority: bb.Priority,
    Type: bb.Kind,
    @@ -61,10 +66,22 @@
    }
    return yt
    -
    }
    -func (a *Archive) convert() (*youtrack.Project, error) {
    +func (a *Archive) convert(usersMap *UsersMap) (*youtrack.Project, error) {
    + users, err := a.ExtractUsers(usersMap)
    + if err != nil {
    + return nil, err
    + }
    +
    + // create a slice of youtrack users for import
    + ytUsers := make([]*youtrack.User, len(users))
    + i := 0
    + for _, user := range users {
    + ytUsers[i] = user
    + i++
    + }
    +
    // create a string slice of all the components
    components := make([]string, len(a.Components))
    for i := 0; i < len(a.Components); i++ {
    @@ -80,10 +97,11 @@
    // convert all of the issues
    issues := make([]*youtrack.Issue, len(a.Issues))
    for i := 0; i < len(a.Issues); i++ {
    - issues[i] = a.convertIssue(a.Issues[i])
    + issues[i] = a.convertIssue(a.Issues[i], users)
    }
    project := &youtrack.Project{
    + Users: ytUsers,
    Subsystems: components,
    Versions: versions,
    Issues: issues,
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/bitbucket/usersmap.go Tue Dec 31 01:08:27 2019 -0600
    @@ -0,0 +1,57 @@
    +package bitbucket
    +
    +import (
    + "encoding/json"
    + "fmt"
    + "io/ioutil"
    +
    + "hg.sr.ht/~grim/youtrack-import/youtrack"
    +)
    +
    +type User struct {
    + FullName string `json:"fullName"`
    + Login string `json:"login"`
    + Email string `json:"email"`
    +}
    +
    +func (u *User) ToYoutrack() *youtrack.User {
    + return youtrack.NewUser(u.Login, u.FullName, u.Email)
    +}
    +
    +type UsersMapData struct {
    + Users []User `json:"users"`
    +}
    +
    +type UsersMap struct {
    + users map[string]User
    +}
    +
    +func NewUsersMap(filename string) (*UsersMap, error) {
    + data, err := ioutil.ReadFile(filename)
    + if err != nil {
    + return nil, err
    + }
    +
    + d := UsersMapData{}
    + if err := json.Unmarshal(data, &d); err != nil {
    + return nil, err
    + }
    +
    + m := &UsersMap{
    + users: map[string]User{},
    + }
    +
    + for _, user := range d.Users {
    + m.users[user.FullName] = user
    + }
    +
    + return m, nil
    +}
    +
    +func (m *UsersMap) Find(displayName string) (User, error) {
    + if user, found := m.users[displayName]; found {
    + return user, nil
    + }
    +
    + return User{}, fmt.Errorf("no user found for %q", displayName)
    +}
    --- a/build-and-run Mon Dec 30 20:17:16 2019 -0600
    +++ b/build-and-run Tue Dec 31 01:08:27 2019 -0600
    @@ -4,5 +4,5 @@
    echo -n "deleting project \"${1}\" ... "
    curl -X DELETE -H "Authorization: Bearer ${YOUTRACK_TOKEN}" "${YOUTRACK_URL}rest/admin/project/${1}"
    echo "done."
    -./youtrack-import --project-id="${1}" --project-name="${2}" bitbucket "${3}"
    +./youtrack-import --project-id="${1}" --project-name="${2}" bitbucket "${3}" "${4}"
    --- a/youtrack/client.go Mon Dec 30 20:17:16 2019 -0600
    +++ b/youtrack/client.go Tue Dec 31 01:08:27 2019 -0600
    @@ -41,7 +41,7 @@
    }
    }
    - fmt.Printf("failed to find an acceptable status code\n")
    + fmt.Printf("failed to find an acceptable status code: got %d\n", resp.StatusCode)
    body, err := ioutil.ReadAll(resp.Body)
    if err != nil {
    --- a/youtrack/import.go Mon Dec 30 20:17:16 2019 -0600
    +++ b/youtrack/import.go Tue Dec 31 01:08:27 2019 -0600
    @@ -41,6 +41,11 @@
    return err
    }
    + // create the users
    + if err := client.ImportUsers(p); err != nil {
    + return err
    + }
    +
    // create the issues
    if err := client.ImportIssues(p); err != nil {
    return err
    --- a/youtrack/issue.go Mon Dec 30 20:17:16 2019 -0600
    +++ b/youtrack/issue.go Tue Dec 31 01:08:27 2019 -0600
    @@ -9,7 +9,7 @@
    )
    type Issue struct {
    - Number int `yt"numberInProject"`
    + Number int `yt:"numberInProject"`
    Summary string `yt:"summary"`
    Description string `yt:"description"`
    Created time.Time `yt:"created"`
    @@ -85,7 +85,7 @@
    // convert the datastructure to an xml string
    data, err := xml.Marshal(issues)
    if err != nil {
    - return fmt.Errorf("wut: %v", err)
    + return err
    }
    // create the reader for the put request
    @@ -97,7 +97,7 @@
    putBody,
    )
    if err != nil {
    - return fmt.Errorf("wtf %v", err)
    + return err
    }
    defer resp.Body.Close()
    @@ -107,10 +107,16 @@
    report := xmlImportReport{}
    if err := xml.NewDecoder(resp.Body).Decode(&report); err != nil {
    - return fmt.Errorf("repo: %v", err)
    + return err
    }
    - fmt.Printf("%#v\n", report)
    + if errs := report.Validate(); len(errs) > 0 {
    + for _, err := range errs {
    + fmt.Printf("%v\n", err)
    + }
    +
    + return fmt.Errorf("error importing issues")
    + }
    return nil
    }
    --- a/youtrack/project.go Mon Dec 30 20:17:16 2019 -0600
    +++ b/youtrack/project.go Tue Dec 31 01:08:27 2019 -0600
    @@ -9,6 +9,8 @@
    Name string
    LeadLogin string
    + Users []*User
    +
    Subsystems []string
    Versions []string
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/youtrack/user.go Tue Dec 31 01:08:27 2019 -0600
    @@ -0,0 +1,129 @@
    +package youtrack
    +
    +import (
    + "bytes"
    + "encoding/xml"
    + "fmt"
    + "net/http"
    +)
    +
    +type User struct {
    + Login string
    + FullName string
    + Email string
    + Jabber string
    +}
    +
    +func NewUser(login, fullName, email string) *User {
    + return &User{
    + Login: login,
    + FullName: fullName,
    + Email: email,
    + }
    +}
    +
    +func (u *User) encode() xmlUser {
    + return xmlUser{
    + Login: u.Login,
    + FullName: u.FullName,
    + Email: u.Email,
    + }
    +}
    +
    +func (c *Client) importUsersRange(u []*User, s, e int) []error {
    + users := &xmlUsers{}
    +
    + users.Users = make([]xmlUser, e-s)
    +
    + for i := s; i < e; i++ {
    + users.Users[i-s] = u[i].encode()
    + }
    +
    + data, err := xml.Marshal(users)
    + if err != nil {
    + return []error{err}
    + }
    +
    + // create the body
    + putBody := bytes.NewReader(data)
    +
    + resp, err := c.xmlRequest(
    + http.MethodPut,
    + c.uri+"/import/users",
    + putBody,
    + )
    + if err != nil {
    + return []error{err}
    + }
    + defer resp.Body.Close()
    +
    + if err := c.checkStatuses(resp, []int{http.StatusOK, http.StatusBadRequest}); err != nil {
    + return []error{err}
    + }
    +
    + report := xmlImportReport{}
    + if err := xml.NewDecoder(resp.Body).Decode(&report); err != nil {
    + return []error{err}
    + }
    +
    + return report.Validate()
    +}
    +
    +func (c *Client) ImportUsers(p *Project) error {
    + r := 10
    +
    + users := []*User{}
    +
    + // loop through the users we were given and create a new list based on the
    + // ones that we need to create.
    + for _, user := range p.Users {
    + exists, err := c.UserExists(user.Login)
    + if err != nil {
    + return err
    + }
    +
    + if !exists {
    + users = append(users, user)
    + }
    + }
    +
    + for s := 0; s < len(users); s += r {
    + e := s + r
    + if e > len(users) {
    + e = len(users)
    + }
    +
    + fmt.Printf("importing users %d-%d ...", s, e)
    +
    + if errs := c.importUsersRange(users, s, e); len(errs) > 0 {
    + fmt.Printf("failed.\n")
    +
    + for _, err := range errs {
    + fmt.Printf("%v\n", err)
    + }
    +
    + return fmt.Errorf("error importing users")
    + }
    +
    + fmt.Printf("done.\n")
    + }
    +
    + return nil
    +}
    +
    +func (c *Client) UserExists(username string) (bool, error) {
    + resp, err := c.xmlRequest(
    + http.MethodHead,
    + c.uri+"/user/"+username,
    + nil,
    + )
    + if err != nil {
    + return false, err
    + }
    +
    + if resp.StatusCode == http.StatusBadRequest {
    + return false, nil
    + }
    +
    + return true, nil
    +}
    --- a/youtrack/xml.go Mon Dec 30 20:17:16 2019 -0600
    +++ b/youtrack/xml.go Tue Dec 31 01:08:27 2019 -0600
    @@ -2,6 +2,8 @@
    import (
    "encoding/xml"
    + "fmt"
    + "strings"
    "time"
    )
    @@ -66,7 +68,43 @@
    Items []xmlImportReportItem `xml:"item"`
    }
    +func (x *xmlImportReport) Validate() []error {
    + ret := []error{}
    +
    + for _, item := range x.Items {
    + if item.Imported == false {
    + errMsg := []string{}
    +
    + for _, e := range item.Errors {
    + msg := fmt.Sprintf(
    + "field: %q; value: %q: %s",
    + e.FieldName,
    + e.Value,
    + e.Message,
    + )
    +
    + errMsg = append(errMsg, msg)
    + }
    +
    + ret = append(ret, fmt.Errorf("%s", strings.Join(errMsg, "\n")))
    + }
    + }
    +
    + return ret
    +}
    +
    type xmlError struct {
    XMLName xml.Name `xml:"error"`
    Message string `xml:",cdata"`
    }
    +
    +type xmlUser struct {
    + Login string `xml:"login,attr"`
    + FullName string `xml:"fullName,attr,omitempty"`
    + Email string `xml:"email,attr,omitempty"`
    +}
    +
    +type xmlUsers struct {
    + XMLName xml.Name `xml:"list"`
    + Users []xmlUser `xml:"user"`
    +}