grim/youtrack-import

A bunch of tweaks to get things mostly importing
draft
2020-07-08, Gary Kramlich
33ea22a7724f
Parents 0ce6488c1f5f
Children a90131f168a3
A bunch of tweaks to get things mostly importing
5 files changed, 118 insertions(+), 44 deletions(-)
  • +11 -4
    trac/components.go
  • +9 -25
    trac/project.go
  • +79 -13
    trac/tickets.go
  • +12 -0
    trac/users.go
  • +7 -2
    trac/versions.go
    • --- a/trac/components.go Tue Jul 07 23:42:04 2020 -0500
    +++ b/trac/components.go Wed Jul 08 05:40:55 2020 -0500
    @@ -12,7 +12,7 @@
    Description sql.NullString `db:"description"`
    }
    -func (e *environment) loadComponents() ([]component, error) {
    +func (e *environment) loadComponents(users map[string]*youtrack.User, unknownUser string) ([]youtrack.OwnedField, error) {
    components := []component{}
    err := e.db.Select(
    &components,
    @@ -26,13 +26,20 @@
    ORDER BY t.component`,
    )
    - return components, err
    + ytSubsystem := make([]youtrack.OwnedField, len(components))
    + for idx, component := range components {
    + ytSubsystem[idx] = component.toYouTrack(users, unknownUser)
    + }
    +
    + return ytSubsystem, err
    }
    -func (c *component) toYouTrack() youtrack.OwnedField {
    +func (c *component) toYouTrack(users map[string]*youtrack.User, unknownUser string) youtrack.OwnedField {
    + owner := mapUser(convertString(c.Owner), unknownUser, users)
    +
    return youtrack.OwnedField{
    Value: c.Name,
    Description: convertString(c.Description),
    - Owner: convertString(c.Owner),
    + Owner: owner,
    }
    }
    --- a/trac/project.go Tue Jul 07 23:42:04 2020 -0500
    +++ b/trac/project.go Wed Jul 08 05:40:55 2020 -0500
    @@ -6,7 +6,7 @@
    "keep.imfreedom.org/grim/youtrack-import/youtrack"
    )
    -func createProject(e *environment, importUsersFile, defaultUser string) (*youtrack.Project, error) {
    +func createProject(e *environment, importUsersFile, unknownUser string) (*youtrack.Project, error) {
    users, err := e.loadUsers(importUsersFile)
    if err != nil {
    return nil, err
    @@ -14,39 +14,25 @@
    fmt.Printf("loaded %d users\n", len(users))
    - versions, err := e.loadVersions()
    + ytVersions, err := e.loadVersions()
    if err != nil {
    return nil, err
    }
    - ytVersions := make([]youtrack.Version, len(versions))
    - for idx, version := range versions {
    - ytVersions[idx] = version.toYouTrack()
    - }
    + fmt.Printf("loaded %d versions\n", len(ytVersions))
    - components, err := e.loadComponents()
    + ytSubsystems, err := e.loadComponents(users, unknownUser)
    if err != nil {
    return nil, err
    }
    - ytSubsystems := make([]youtrack.OwnedField, len(components))
    - for idx, component := range components {
    - ytSubsystems[idx] = component.toYouTrack()
    - }
    + fmt.Printf("loaded %d subsystems\n", len(ytSubsystems))
    - tracTickets, err := e.loadTickets()
    + ytIssues, err := e.loadTickets(users, unknownUser)
    if err != nil {
    return nil, err
    }
    + fmt.Printf("loaded %d issues\n", len(ytIssues))
    - issues := make([]*youtrack.Issue, len(tracTickets))
    - for idx, ticket := range tracTickets {
    - issue, err := ticket.toYoutrack(defaultUser, users)
    - if err != nil {
    - return nil, err
    - }
    -
    - issues[idx] = issue
    - }
    -
    + // build a new slice of users that only includes the ones we're importing.
    sliceUsers := make([]*youtrack.User, len(users))
    idx := 0
    for _, user := range users {
    @@ -55,13 +41,11 @@
    }
    project := &youtrack.Project{
    - Issues: issues,
    + Issues: ytIssues,
    Users: sliceUsers,
    Versions: ytVersions,
    Subsystems: ytSubsystems,
    }
    - fmt.Printf("Project: has %d issues\n", len(issues))
    -
    return project, nil
    }
    --- a/trac/tickets.go Tue Jul 07 23:42:04 2020 -0500
    +++ b/trac/tickets.go Wed Jul 08 05:40:55 2020 -0500
    @@ -2,6 +2,7 @@
    import (
    "database/sql"
    + "fmt"
    "strings"
    "keep.imfreedom.org/grim/youtrack-import/youtrack"
    @@ -72,14 +73,10 @@
    }
    )
    -func (t *ticket) toYoutrack(defaultUser string, users map[string]*youtrack.User) (*youtrack.Issue, error) {
    +func (t *ticket) toYoutrack(users map[string]*youtrack.User, unknownUser string) (*youtrack.Issue, error) {
    convertUsername := func(sqluser sql.NullString) string {
    username := convertString(sqluser)
    - if user, found := users[username]; found {
    - return user.Login
    - }
    -
    - return defaultUser
    + return mapUser(username, unknownUser, users)
    }
    issue := &youtrack.Issue{
    @@ -98,34 +95,93 @@
    State: stateMap[t.Status],
    }
    + // There are some instances were reporters are null in our trac database, so
    + // convert them to the unknownUser.
    + if issue.Reporter == "" {
    + issue.Reporter = unknownUser
    + }
    +
    // convert the trac cc to a string slice of known youtrack users
    watchers := strings.Split(convertString(t.CarbonCopy), ",")
    ytWatchers := []string{}
    for _, watcher := range watchers {
    - if ytUser, found := users[watcher]; found {
    - ytWatchers = append(ytWatchers, ytUser.Login)
    + login := mapUser(watcher, unknownUser, users)
    + if login != "" && login != unknownUser {
    + ytWatchers = append(ytWatchers, login)
    }
    }
    issue.Watchers = ytWatchers
    // walk through the changes and update fields as necessary
    + // due to the way trac's data is stored, we need to condense state updates
    + // as we iterate the comments and then only "commit" the change when it has
    + // a new author/timestamp or when we're looking at the last one.
    + var comment *youtrack.Comment
    for _, change := range t.Changes {
    + newComment := &youtrack.Comment{
    + Author: convertUsername(change.Author),
    + Created: convertTime(t.Time),
    + Updated: convertTime(t.Time),
    + Markdown: false,
    + }
    +
    + // There's some oddness in our trac where we have a few changes where
    + // the author is null, so if it's empty string, set it to the
    + // unknownUser.
    + if newComment.Author == "" {
    + newComment.Author = unknownUser
    + }
    +
    + if comment == nil {
    + comment = newComment
    + } else {
    + if comment.Author != newComment.Author || comment.Created != newComment.Created {
    + issue.Comments = append(issue.Comments, comment)
    + comment = newComment
    + }
    + }
    +
    + text := ""
    switch change.Field {
    + case "comment":
    + // if the comment already has text, add a newline to help make the
    + // actual comment text stand out.
    + if comment.Text != "" {
    + text += "\n"
    + }
    +
    + text += convertString(change.NewValue)
    + case "description":
    + issue.UpdatedBy = convertUsername(change.Author)
    + text = "description: modified\n"
    case "resolution":
    if convertString(change.NewValue) == "fixed" {
    issue.Resolved = convertTime(change.Time)
    }
    case "summary":
    - issue.UpdatedBy = convertString(change.Author)
    - case "description":
    - issue.UpdatedBy = convertString(change.Author)
    + issue.UpdatedBy = convertUsername(change.Author)
    }
    +
    + if text == "" {
    + text = fmt.Sprintf(
    + "%s: %s -> %s\n",
    + change.Field,
    + convertString(change.OldValue),
    + convertString(change.NewValue),
    + )
    + }
    +
    + comment.Text += text
    + }
    +
    + if comment != nil {
    + issue.Comments = append(issue.Comments, comment)
    }
    return issue, nil
    }
    -func (e *environment) loadTickets() ([]*ticket, error) {
    +func (e *environment) loadTickets(users map[string]*youtrack.User, unknownUser string) ([]*youtrack.Issue, error) {
    var err error
    tickets := []*ticket{}
    @@ -156,5 +212,15 @@
    }
    }
    - return tickets, nil
    + issues := make([]*youtrack.Issue, len(tickets))
    + for idx, ticket := range tickets {
    + issue, err := ticket.toYoutrack(users, unknownUser)
    + if err != nil {
    + return nil, err
    + }
    +
    + issues[idx] = issue
    + }
    +
    + return issues, nil
    }
    --- a/trac/users.go Tue Jul 07 23:42:04 2020 -0500
    +++ b/trac/users.go Wed Jul 08 05:40:55 2020 -0500
    @@ -57,3 +57,15 @@
    return users, nil
    }
    +
    +func mapUser(user, unknownUser string, users map[string]*youtrack.User) string {
    + if user == "" {
    + return user
    + }
    +
    + if ytUser, found := users[user]; found {
    + return ytUser.Login
    + }
    +
    + return unknownUser
    +}
    --- a/trac/versions.go Tue Jul 07 23:42:04 2020 -0500
    +++ b/trac/versions.go Wed Jul 08 05:40:55 2020 -0500
    @@ -12,14 +12,19 @@
    Description sql.NullString `db:"description"`
    }
    -func (e *environment) loadVersions() ([]version, error) {
    +func (e *environment) loadVersions() ([]youtrack.Version, error) {
    versions := []version{}
    err := e.db.Select(
    &versions,
    "SELECT * from version ORDER BY name DESC",
    )
    - return versions, err
    + ytVersions := make([]youtrack.Version, len(versions))
    + for idx, version := range versions {
    + ytVersions[idx] = version.toYouTrack()
    + }
    +
    + return ytVersions, err
    }
    func (v *version) toYouTrack() youtrack.Version {