grim/youtrack-import
Clone
Summary
Browse
Changes
Graph
Add support for auto linking tickets. Fixes YI-31
draft
2020-08-11, Gary Kramlich
ff8d512f4f72
Add support for auto linking tickets. Fixes YI-31
package
trac
import
(
"crypto/sha1"
"database/sql"
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"keep.imfreedom.org/grim/youtrack-import/youtrack"
)
type
ticket
struct
{
ID
int
`db:"id"`
Type
string
`db:"type"`
Time
int64
`db:"time"`
ChangeTime
int64
`db:"changetime"`
Component
sql
.
NullString
`db:"component"`
Severity
sql
.
NullString
`db:"severity"`
Priority
sql
.
NullString
`db:"priority"`
Owner
sql
.
NullString
`db:"owner"`
Reporter
sql
.
NullString
`db:"reporter"`
CarbonCopy
sql
.
NullString
`db:"cc"`
Version
sql
.
NullString
`db:"version"`
Milestone
sql
.
NullString
`db:"milestone"`
Status
string
`db:"status"`
Resolution
sql
.
NullString
`db:"resolution"`
Summary
string
`db:"summary"`
Description
string
`db:"description"`
Keywords
sql
.
NullString
`db:"keywords"`
Changes
[]
*
ticketChange
Customs
[]
*
ticketCustom
Attachments
[]
*
ticketAttachment
}
type
ticketChange
struct
{
Ticket
int
`db:"ticket"`
Time
int64
`db:"time"`
Author
sql
.
NullString
`db:"author"`
Field
string
`db:"field"`
OldValue
sql
.
NullString
`db:"oldvalue"`
NewValue
sql
.
NullString
`db:"newvalue"`
}
type
ticketCustom
struct
{
Ticket
int
`db:"ticket"`
Name
sql
.
NullString
`db:"name"`
Value
sql
.
NullString
`db:"value"`
}
type
ticketAttachment
struct
{
Filename
string
`db:"filename"`
Size
int64
`db:"size"`
Time
int64
`db:"time"`
Description
sql
.
NullString
`db:"description"`
Author
string
`db:"author"`
path
string
}
var
(
typeMap
=
map
[
string
]
string
{
"defect"
:
"Bug"
,
"enhancement"
:
"Feature"
,
"patch"
:
"Feature"
,
"plugin request"
:
"Feature"
,
"rejected_patch"
:
"Feature"
,
"task"
:
"Task"
,
"translation"
:
"Feature"
,
}
stateMap
=
map
[
string
]
string
{
"closed"
:
"Fixed"
,
"new"
:
"Submitted"
,
"pending"
:
"To be discussed"
,
}
priorityMap
=
map
[
string
]
string
{
""
:
"Normal"
,
"blocker"
:
"Show-stopper"
,
"critical"
:
"Critical"
,
"major"
:
"Major"
,
"minor"
:
"Normal"
,
"trivial"
:
"Minor"
,
}
// trac uses a crap regex for extensions that's missing - among other
// things, so we just use the same regex to figure out the extension for
// attachments.
attachmentRegex
=
regexp
.
MustCompile
(
`\.[A-Za-z0-9]+$`
)
// trac comments that are replies specify usernames which we can't/don't
// want to put in our resulting comments.
commentReplyingRegex
=
regexp
.
MustCompile
(
`(^|\r\n)(?:(?:\> )*Replying to (?:[^]]+)\]:(\r\n)?)+`
)
// trac's comments use {{{ and }}} for code blocks. We need to convert them.
// Also, you can have a code block in a block quote which is why we have
// (?:\> )* to make sure that'll be supported.
commentCodeBlockRegex
=
regexp
.
MustCompile
(
`(?s)((?:^|\r\n)(?:\> )*)(?:\s*{{{\s*)(\r\n.*?)(?: *}}}\s*?)(\r\n|$)`
)
// trac supports [[br]] for an explicit line break, we repalce it with a
// <br/> tag.
breakRegex
=
regexp
.
MustCompile
(
`(?i)\[\[br\]\]`
)
// trac's inline code is done via "``" and "{{{}}}", since markdown already
// handles "``" we just need to convert "{{{}}}" to "``"
inlineCodeRegex
=
regexp
.
MustCompile
(
`{{{(.+?)}}}`
)
// bold/italic suck, these deal with it.
boldItalicRegex
=
regexp
.
MustCompile
(
`'''''(.+?)'''''`
)
boldRegex
=
regexp
.
MustCompile
(
`'''(.+?)'''`
)
italicRegex
=
regexp
.
MustCompile
(
`''(.+?)''`
)
// ticket link regex
ticketLinkRegex
=
regexp
.
MustCompile
(
`#(\d+)([^\w])`
)
)
func
convertComment
(
projectID
,
comment
string
)
string
{
if
strings
.
Contains
(
comment
,
"{{{"
)
{
// inline code uses the same syntax so we need to do code blocks
// first.
comment
=
commentCodeBlockRegex
.
ReplaceAllString
(
comment
,
"$1```$2```$3"
)
comment
=
inlineCodeRegex
.
ReplaceAllString
(
comment
,
"`$1`"
)
}
// repalce all [[br]]'s with <br/>
comment
=
breakRegex
.
ReplaceAllString
(
comment
,
"<br/>"
)
// replace bold italics with 3 *'s because that's how markdown does it...
comment
=
boldItalicRegex
.
ReplaceAllString
(
comment
,
"***$1***"
)
// replace bold with 2 *'s because that's bold in markdown...
comment
=
boldRegex
.
ReplaceAllString
(
comment
,
"**$1**"
)
// replace italic with 1 *'s because that's italic in markdown...
comment
=
italicRegex
.
ReplaceAllString
(
comment
,
"*$1*"
)
// replace any ticket link references
comment
=
ticketLinkRegex
.
ReplaceAllString
(
comment
,
projectID
+
"-$1$2"
)
return
comment
}
func
(
tc
*
ticketChange
)
cleanNewValue
(
projectID
string
)
string
{
if
tc
.
Field
==
"comment"
{
newComment
:=
convertString
(
tc
.
NewValue
)
if
strings
.
Contains
(
convertString
(
tc
.
OldValue
),
"."
)
{
newComment
=
commentReplyingRegex
.
ReplaceAllString
(
newComment
,
""
)
}
return
convertComment
(
projectID
,
newComment
)
}
return
tc
.
NewValue
.
String
}
func
(
t
*
ticket
)
toYoutrack
(
projectID
string
,
users
map
[
string
]
*
youtrack
.
User
,
unknownUser
string
)
(
*
youtrack
.
Issue
,
error
)
{
convertUsername
:=
func
(
sqluser
sql
.
NullString
)
string
{
username
:=
convertString
(
sqluser
)
return
mapUser
(
username
,
unknownUser
,
users
)
}
addComment
:=
func
(
issue
*
youtrack
.
Issue
,
description
string
,
comment
*
youtrack
.
Comment
)
{
// Add the description at the end to make sure all the field changes
// are always listed before it.
if
description
!=
""
{
if
comment
.
Text
!=
""
{
comment
.
Text
+=
"\n\n"
}
comment
.
Text
+=
description
}
// double check if there is no comment.text and if so dump in a
// *no description*.
if
comment
.
Text
==
""
{
comment
.
Text
=
"*no description*"
}
// our changes were loaded from the database ordered by timestamp,
// so if we hit a new time stamp, we're in a new change.
issue
.
Comments
=
append
(
issue
.
Comments
,
comment
)
}
issue
:=
&
youtrack
.
Issue
{
Number
:
t
.
ID
,
Summary
:
t
.
Summary
,
Description
:
convertComment
(
projectID
,
t
.
Description
),
Created
:
convertTime
(
t
.
Time
),
Updated
:
convertTime
(
t
.
ChangeTime
),
Reporter
:
convertUsername
(
t
.
Reporter
),
Assignee
:
convertUsername
(
t
.
Owner
),
Markdown
:
true
,
AffectedVersion
:
convertString
(
t
.
Version
),
FixVersion
:
convertString
(
t
.
Milestone
),
Subsystem
:
convertString
(
t
.
Component
),
Priority
:
priorityMap
[
convertString
(
t
.
Priority
)],
Type
:
typeMap
[
t
.
Type
],
State
:
stateMap
[
t
.
Status
],
}
// There are some instances were reporters are null in our trac database, so
// convert them to the unknownUser.
if
issue
.
Reporter
==
""
{
issue
.
Reporter
=
unknownUser
}
// convert the trac cc to a string slice of known youtrack users
watchers
:=
strings
.
Split
(
convertString
(
t
.
CarbonCopy
),
","
)
ytWatchers
:=
[]
string
{}
for
_
,
watcher
:=
range
watchers
{
login
:=
mapUser
(
watcher
,
unknownUser
,
users
)
if
login
!=
""
&&
login
!=
unknownUser
{
ytWatchers
=
append
(
ytWatchers
,
login
)
}
}
issue
.
Watchers
=
ytWatchers
// walk through the changes and update fields as necessary
// due to the way trac's data is stored, we need to condense state updates
// as we iterate the comments and then only "commit" the change when it has
// a new author/timestamp or when we're looking at the last one.
var
comment
*
youtrack
.
Comment
var
description
string
for
_
,
change
:=
range
t
.
Changes
{
if
comment
!=
nil
&&
convertTime
(
change
.
Time
)
!=
comment
.
Created
{
addComment
(
issue
,
description
,
comment
)
comment
=
nil
description
=
""
}
if
comment
==
nil
{
comment
=
&
youtrack
.
Comment
{
Author
:
convertUsername
(
change
.
Author
),
Created
:
convertTime
(
change
.
Time
),
Updated
:
convertTime
(
change
.
Time
),
Markdown
:
true
,
}
// There's some oddness in our trac where we have a few changes where
// the author is null, so if it's empty string, set it to the
// unknownUser.
if
comment
.
Author
==
""
{
comment
.
Author
=
unknownUser
}
}
text
:=
""
switch
change
.
Field
{
case
"comment"
:
description
=
change
.
cleanNewValue
(
projectID
)
continue
case
"description"
:
issue
.
UpdatedBy
=
convertUsername
(
change
.
Author
)
text
=
"description: modified\n"
case
"resolution"
:
if
convertString
(
change
.
NewValue
)
==
"fixed"
{
issue
.
Resolved
=
convertTime
(
change
.
Time
)
}
case
"summary"
:
issue
.
UpdatedBy
=
convertUsername
(
change
.
Author
)
}
if
text
==
""
{
oldValue
:=
convertString
(
change
.
OldValue
)
if
oldValue
==
""
{
oldValue
=
"None"
}
newValue
:=
convertString
(
change
.
NewValue
)
if
newValue
==
""
{
newValue
=
"None"
}
fieldName
:=
change
.
Field
switch
fieldName
{
case
"owner"
:
fieldName
=
"Assignee"
}
if
strings
.
HasPrefix
(
change
.
Field
,
"_comment"
)
{
text
=
"*previously edited comment discarded*"
}
else
{
text
=
fmt
.
Sprintf
(
"%s: %s ⟶ %s\n"
,
fieldName
,
oldValue
,
newValue
,
)
}
}
comment
.
Text
+=
text
}
if
comment
!=
nil
{
addComment
(
issue
,
description
,
comment
)
comment
=
nil
description
=
""
}
// add all of the attachments
issue
.
Attachments
=
make
([]
*
youtrack
.
Attachment
,
len
(
t
.
Attachments
))
for
idx
,
attachment
:=
range
t
.
Attachments
{
issue
.
Attachments
[
idx
]
=
attachment
.
toYouTrack
(
t
,
users
,
unknownUser
)
// create a comment for each attachment
description
:=
fmt
.
Sprintf
(
"Attachment: [%s](%s) added.\n\n%s"
,
attachment
.
Filename
,
attachment
.
Filename
,
convertString
(
attachment
.
Description
),
)
newComment
:=
&
youtrack
.
Comment
{
Author
:
mapUser
(
attachment
.
Author
,
unknownUser
,
users
),
Created
:
convertTime
(
attachment
.
Time
),
Updated
:
convertTime
(
attachment
.
Time
),
Text
:
description
,
Markdown
:
true
,
}
issue
.
Comments
=
append
(
issue
.
Comments
,
newComment
)
}
// sort comments by time so that all of the attachment comments appear in
// the correct order.
sort
.
Slice
(
issue
.
Comments
,
func
(
i
,
j
int
)
bool
{
return
issue
.
Comments
[
i
].
Created
.
Before
(
issue
.
Comments
[
j
].
Created
)
})
return
issue
,
nil
}
func
(
e
*
environment
)
loadTickets
(
projectID
string
,
users
map
[
string
]
*
youtrack
.
User
,
unknownUser
string
)
([]
*
youtrack
.
Issue
,
error
)
{
var
err
error
tickets
:=
[]
*
ticket
{}
err
=
e
.
db
.
Select
(
&
tickets
,
"SELECT * FROM ticket ORDER BY id"
)
if
err
!=
nil
{
return
nil
,
err
}
for
_
,
ticket
:=
range
tickets
{
// Load all of the ticket changes.
err
=
e
.
db
.
Select
(
&
ticket
.
Changes
,
`SELECT * FROM ticket_change WHERE ticket = $1 ORDER BY time`
,
ticket
.
ID
,
)
if
err
!=
nil
{
return
nil
,
err
}
// Load all of the ticket custom fields.
err
=
e
.
db
.
Select
(
&
ticket
.
Customs
,
`SELECT * FROM ticket_custom WHERE ticket = $1`
,
ticket
.
ID
,
)
if
err
!=
nil
{
return
nil
,
err
}
// Load all attachments
if
err
:=
e
.
loadAttachments
(
ticket
,
users
,
unknownUser
);
err
!=
nil
{
return
nil
,
err
}
}
issues
:=
make
([]
*
youtrack
.
Issue
,
len
(
tickets
))
for
idx
,
ticket
:=
range
tickets
{
issue
,
err
:=
ticket
.
toYoutrack
(
projectID
,
users
,
unknownUser
)
if
err
!=
nil
{
return
nil
,
err
}
issues
[
idx
]
=
issue
}
return
issues
,
nil
}
func
(
e
*
environment
)
loadAttachments
(
t
*
ticket
,
users
map
[
string
]
*
youtrack
.
User
,
unknownUser
string
)
error
{
err
:=
e
.
db
.
Select
(
&
t
.
Attachments
,
`SELECT
filename, size, time, description, author
FROM trac_pidgin.attachment
WHERE id=$1 AND type='ticket'
ORDER BY time ASC`
,
t
.
ID
,
)
if
err
!=
nil
{
return
err
}
// we figure out all the filenames here as we have the environment which we
// need to figure out the path.
for
_
,
attachment
:=
range
t
.
Attachments
{
// figure out the path for the filename
parent_sum
:=
fmt
.
Sprintf
(
"%02x"
,
sha1
.
Sum
([]
byte
(
fmt
.
Sprintf
(
"%d"
,
t
.
ID
))))
filename_sum
:=
fmt
.
Sprintf
(
"%02x"
,
sha1
.
Sum
([]
byte
(
attachment
.
Filename
)))
ext
:=
attachmentRegex
.
Find
([]
byte
(
attachment
.
Filename
))
attachment
.
path
=
filepath
.
Join
(
e
.
path
,
"files"
,
"attachments"
,
"ticket"
,
parent_sum
[:
3
],
parent_sum
,
filename_sum
+
string
(
ext
),
)
}
return
nil
}
func
(
ta
*
ticketAttachment
)
toYouTrack
(
t
*
ticket
,
users
map
[
string
]
*
youtrack
.
User
,
unknownUser
string
)
*
youtrack
.
Attachment
{
// map the owner
author
:=
mapUser
(
ta
.
Author
,
unknownUser
,
users
)
// create the youtrack attachment
a
:=
youtrack
.
NewAttachment
(
t
.
ID
,
author
,
convertTime
(
ta
.
Time
))
// open the file from the attachment which is already an absolute path on
// disk.
file
,
err
:=
os
.
Open
(
ta
.
path
)
if
err
!=
nil
{
panic
(
err
)
}
a
.
AddFile
(
file
,
ta
.
Filename
)
return
a
}