grim/youtrack-import

Split the version field into AffectedVersion and FixVersion while having trac and bitbucket use AffectedVersion. Also put the trac milestone into the Fix Version.
draft
2020-08-11, Gary Kramlich
faf8f5a4120f
Split the version field into AffectedVersion and FixVersion while having trac and bitbucket use AffectedVersion. Also put the trac milestone into the Fix Version.
package trac
import (
"crypto/sha1"
"database/sql"
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"keep.imfreedom.org/grim/youtrack-import/youtrack"
)
type ticket struct {
ID int `db:"id"`
Type string `db:"type"`
Time int64 `db:"time"`
ChangeTime int64 `db:"changetime"`
Component sql.NullString `db:"component"`
Severity sql.NullString `db:"severity"`
Priority sql.NullString `db:"priority"`
Owner sql.NullString `db:"owner"`
Reporter sql.NullString `db:"reporter"`
CarbonCopy sql.NullString `db:"cc"`
Version sql.NullString `db:"version"`
Milestone sql.NullString `db:"milestone"`
Status string `db:"status"`
Resolution sql.NullString `db:"resolution"`
Summary string `db:"summary"`
Description string `db:"description"`
Keywords sql.NullString `db:"keywords"`
Changes []*ticketChange
Customs []*ticketCustom
Attachments []*ticketAttachment
}
type ticketChange struct {
Ticket int `db:"ticket"`
Time int64 `db:"time"`
Author sql.NullString `db:"author"`
Field string `db:"field"`
OldValue sql.NullString `db:"oldvalue"`
NewValue sql.NullString `db:"newvalue"`
}
type ticketCustom struct {
Ticket int `db:"ticket"`
Name sql.NullString `db:"name"`
Value sql.NullString `db:"value"`
}
type ticketAttachment struct {
Filename string `db:"filename"`
Size int64 `db:"size"`
Time int64 `db:"time"`
Description sql.NullString `db:"description"`
Author string `db:"author"`
path string
}
var (
typeMap = map[string]string{
"defect": "Bug",
"enhancement": "Feature",
"patch": "Feature",
"plugin request": "Feature",
"rejected_patch": "Feature",
"task": "Task",
"translation": "Feature",
}
stateMap = map[string]string{
"closed": "Fixed",
"new": "Submitted",
"pending": "To be discussed",
}
priorityMap = map[string]string{
"": "Normal",
"blocker": "Show-stopper",
"critical": "Critical",
"major": "Major",
"minor": "Normal",
"trivial": "Minor",
}
// trac uses a crap regex for extensions that's missing - among other
// things, so we just use the same regex to figure out the extension for
// attachments.
attachmentRegex = regexp.MustCompile(`\.[A-Za-z0-9]+$`)
// trac comments that are replies specify usernames which we can't/don't
// want to put in our resulting comments.
commentReplyingRegex = regexp.MustCompile(`(^|\r\n)(?:(?:\> )*Replying to (?:[^]]+)\]:(\r\n)?)+`)
// trac's comments use {{{ and }}} for code blocks. We need to convert them.
// Also, you can have a code block in a block quote which is why we have
// (?:\> )* to make sure that'll be supported.
commentCodeBlockRegex = regexp.MustCompile(`(?s)((?:^|\r\n)(?:\> )*)(?:\s*{{{\s*)(\r\n.*?)(?: *}}}\s*?)(\r\n|$)`)
// trac supports [[br]] for an explicit line break, we repalce it with a
// <br/> tag.
breakRegex = regexp.MustCompile(`(?i)\[\[br\]\]`)
// trac's inline code is done via "``" and "{{{}}}", since markdown already
// handles "``" we just need to convert "{{{}}}" to "``"
inlineCodeRegex = regexp.MustCompile(`{{{(.+?)}}}`)
// bold/italic suck, these deal with it.
boldItalicRegex = regexp.MustCompile(`'''''(.+?)'''''`)
boldRegex = regexp.MustCompile(`'''(.+?)'''`)
italicRegex = regexp.MustCompile(`''(.+?)''`)
)
func convertComment(comment string) string {
if strings.Contains(comment, "{{{") {
// inline code uses the same syntax so we need to do code blocks
// first.
comment = commentCodeBlockRegex.ReplaceAllString(comment, "$1```$2```$3")
comment = inlineCodeRegex.ReplaceAllString(comment, "`$1`")
}
// repalce all [[br]]'s with <br/>
comment = breakRegex.ReplaceAllString(comment, "<br/>")
// replace bold italics with 3 *'s because that's how markdown does it...
comment = boldItalicRegex.ReplaceAllString(comment, "***$1***")
// replace bold with 2 *'s because that's bold in markdown...
comment = boldRegex.ReplaceAllString(comment, "**$1**")
// replace italic with 1 *'s because that's italic in markdown...
comment = italicRegex.ReplaceAllString(comment, "*$1*")
return comment
}
func (tc *ticketChange) cleanNewValue() string {
if tc.Field == "comment" {
newComment := convertString(tc.NewValue)
if strings.Contains(convertString(tc.OldValue), ".") {
newComment = commentReplyingRegex.ReplaceAllString(newComment, "")
}
return convertComment(newComment)
}
return tc.NewValue.String
}
func (t *ticket) toYoutrack(users map[string]*youtrack.User, unknownUser string) (*youtrack.Issue, error) {
convertUsername := func(sqluser sql.NullString) string {
username := convertString(sqluser)
return mapUser(username, unknownUser, users)
}
addComment := func(issue *youtrack.Issue, description string, comment *youtrack.Comment) {
// Add the description at the end to make sure all the field changes
// are always listed before it.
if description != "" {
if comment.Text != "" {
comment.Text += "\n\n"
}
comment.Text += description
}
// double check if there is no comment.text and if so dump in a
// *no description*.
if comment.Text == "" {
comment.Text = "*no description*"
}
// our changes were loaded from the database ordered by timestamp,
// so if we hit a new time stamp, we're in a new change.
issue.Comments = append(issue.Comments, comment)
}
issue := &youtrack.Issue{
Number: t.ID,
Summary: t.Summary,
Description: convertComment(t.Description),
Created: convertTime(t.Time),
Updated: convertTime(t.ChangeTime),
Reporter: convertUsername(t.Reporter),
Assignee: convertUsername(t.Owner),
Markdown: true,
AffectedVersion: convertString(t.Version),
FixVersion: convertString(t.Milestone),
Subsystem: convertString(t.Component),
Priority: priorityMap[convertString(t.Priority)],
Type: typeMap[t.Type],
State: stateMap[t.Status],
}
// There are some instances were reporters are null in our trac database, so
// convert them to the unknownUser.
if issue.Reporter == "" {
issue.Reporter = unknownUser
}
// convert the trac cc to a string slice of known youtrack users
watchers := strings.Split(convertString(t.CarbonCopy), ",")
ytWatchers := []string{}
for _, watcher := range watchers {
login := mapUser(watcher, unknownUser, users)
if login != "" && login != unknownUser {
ytWatchers = append(ytWatchers, login)
}
}
issue.Watchers = ytWatchers
// walk through the changes and update fields as necessary
// due to the way trac's data is stored, we need to condense state updates
// as we iterate the comments and then only "commit" the change when it has
// a new author/timestamp or when we're looking at the last one.
var comment *youtrack.Comment
var description string
for _, change := range t.Changes {
if comment != nil && convertTime(change.Time) != comment.Created {
addComment(issue, description, comment)
comment = nil
description = ""
}
if comment == nil {
comment = &youtrack.Comment{
Author: convertUsername(change.Author),
Created: convertTime(change.Time),
Updated: convertTime(change.Time),
Markdown: true,
}
// There's some oddness in our trac where we have a few changes where
// the author is null, so if it's empty string, set it to the
// unknownUser.
if comment.Author == "" {
comment.Author = unknownUser
}
}
text := ""
switch change.Field {
case "comment":
description = change.cleanNewValue()
continue
case "description":
issue.UpdatedBy = convertUsername(change.Author)
text = "description: modified\n"
case "resolution":
if convertString(change.NewValue) == "fixed" {
issue.Resolved = convertTime(change.Time)
}
case "summary":
issue.UpdatedBy = convertUsername(change.Author)
}
if text == "" {
oldValue := convertString(change.OldValue)
if oldValue == "" {
oldValue = "None"
}
newValue := convertString(change.NewValue)
if newValue == "" {
newValue = "None"
}
fieldName := change.Field
switch fieldName {
case "owner":
fieldName = "Assignee"
}
if strings.HasPrefix(change.Field, "_comment") {
text = "*previously edited comment discarded*"
} else {
text = fmt.Sprintf(
"%s: %s ⟶ %s\n",
fieldName,
oldValue,
newValue,
)
}
}
comment.Text += text
}
if comment != nil {
addComment(issue, description, comment)
comment = nil
description = ""
}
// add all of the attachments
issue.Attachments = make([]*youtrack.Attachment, len(t.Attachments))
for idx, attachment := range t.Attachments {
issue.Attachments[idx] = attachment.toYouTrack(t, users, unknownUser)
// create a comment for each attachment
description := fmt.Sprintf(
"Attachment: [%s](%s) added.\n\n%s",
attachment.Filename,
attachment.Filename,
convertString(attachment.Description),
)
newComment := &youtrack.Comment{
Author: mapUser(attachment.Author, unknownUser, users),
Created: convertTime(attachment.Time),
Updated: convertTime(attachment.Time),
Text: description,
Markdown: true,
}
issue.Comments = append(issue.Comments, newComment)
}
// sort comments by time so that all of the attachment comments appear in
// the correct order.
sort.Slice(issue.Comments, func(i, j int) bool {
return issue.Comments[i].Created.Before(issue.Comments[j].Created)
})
return issue, nil
}
func (e *environment) loadTickets(users map[string]*youtrack.User, unknownUser string) ([]*youtrack.Issue, error) {
var err error
tickets := []*ticket{}
err = e.db.Select(&tickets, "SELECT * FROM ticket ORDER BY id")
if err != nil {
return nil, err
}
for _, ticket := range tickets {
// Load all of the ticket changes.
err = e.db.Select(
&ticket.Changes,
`SELECT * FROM ticket_change WHERE ticket = $1 ORDER BY time`,
ticket.ID,
)
if err != nil {
return nil, err
}
// Load all of the ticket custom fields.
err = e.db.Select(
&ticket.Customs,
`SELECT * FROM ticket_custom WHERE ticket = $1`,
ticket.ID,
)
if err != nil {
return nil, err
}
// Load all attachments
if err := e.loadAttachments(ticket, users, unknownUser); err != nil {
return nil, err
}
}
issues := make([]*youtrack.Issue, len(tickets))
for idx, ticket := range tickets {
issue, err := ticket.toYoutrack(users, unknownUser)
if err != nil {
return nil, err
}
issues[idx] = issue
}
return issues, nil
}
func (e *environment) loadAttachments(t *ticket, users map[string]*youtrack.User, unknownUser string) error {
err := e.db.Select(
&t.Attachments,
`SELECT
filename, size, time, description, author
FROM trac_pidgin.attachment
WHERE id=$1 AND type='ticket'
ORDER BY time ASC`,
t.ID,
)
if err != nil {
return err
}
// we figure out all the filenames here as we have the environment which we
// need to figure out the path.
for _, attachment := range t.Attachments {
// figure out the path for the filename
parent_sum := fmt.Sprintf("%02x", sha1.Sum([]byte(fmt.Sprintf("%d", t.ID))))
filename_sum := fmt.Sprintf("%02x", sha1.Sum([]byte(attachment.Filename)))
ext := attachmentRegex.Find([]byte(attachment.Filename))
attachment.path = filepath.Join(
e.path,
"files",
"attachments",
"ticket",
parent_sum[:3],
parent_sum,
filename_sum+string(ext),
)
}
return nil
}
func (ta *ticketAttachment) toYouTrack(t *ticket, users map[string]*youtrack.User, unknownUser string) *youtrack.Attachment {
// map the owner
author := mapUser(ta.Author, unknownUser, users)
// create the youtrack attachment
a := youtrack.NewAttachment(t.ID, author, convertTime(ta.Time))
// open the file from the attachment which is already an absolute path on
// disk.
file, err := os.Open(ta.path)
if err != nil {
panic(err)
}
a.AddFile(file, ta.Filename)
return a
}