grim/youtrack-import

The basics of file attachments are working. Mime types are broken and there's some timing issue yet

2020-01-14, Gary Kramlich
f19a2cb91197
Parents ac0a0ad87f41
Children 4763018b25c8
The basics of file attachments are working. Mime types are broken and there's some timing issue yet
9 files changed, 202 insertions(+), 23 deletions(-)
  • +23 -1
    bitbucket/archive.go
  • +1 -0
    bitbucket/cmd.go
  • +30 -1
    bitbucket/converter.go
  • +3 -0
    bitbucket/types.go
  • +107 -0
    youtrack/attachment.go
  • +5 -2
    youtrack/client.go
  • +5 -0
    youtrack/import.go
  • +6 -19
    youtrack/issue.go
  • +22 -0
    youtrack/xml.go
    • --- a/bitbucket/archive.go Tue Jan 14 01:08:46 2020 -0600
    +++ b/bitbucket/archive.go Tue Jan 14 04:41:41 2020 -0600
    @@ -4,6 +4,7 @@
    "archive/zip"
    "encoding/json"
    "fmt"
    + "io"
    "hg.sr.ht/~grim/youtrack-import/youtrack"
    )
    @@ -13,7 +14,6 @@
    if err != nil {
    return nil, err
    }
    - defer r.Close()
    for _, f := range r.File {
    if f.Name == "db-2.0.json" {
    @@ -28,6 +28,8 @@
    return nil, err
    }
    + archive.zip = r
    +
    return archive, nil
    }
    }
    @@ -35,6 +37,26 @@
    return nil, fmt.Errorf("failed to find db-2.0.json in the archive")
    }
    +func (a *Archive) Close() error {
    + return a.zip.Close()
    +}
    +
    +func (a *Archive) OpenAttachment(path string) (io.ReadCloser, error) {
    + // find the file in the archive
    + for _, f := range a.zip.File {
    + if f.Name == path {
    + r, err := f.Open()
    + if err != nil {
    + return nil, err
    + }
    +
    + return r, nil
    + }
    + }
    +
    + return nil, fmt.Errorf("failed to find attachment %q", path)
    +}
    +
    func (a *Archive) ExtractUsers(usersMap *UsersMap) (map[string]*youtrack.User, error) {
    // key is the id of the user
    users := map[string]*youtrack.User{}
    --- a/bitbucket/cmd.go Tue Jan 14 01:08:46 2020 -0600
    +++ b/bitbucket/cmd.go Tue Jan 14 04:41:41 2020 -0600
    @@ -15,6 +15,7 @@
    if err != nil {
    return err
    }
    + defer archive.Close()
    usersMap, err := NewUsersMap(c.UsersMapFile)
    if err != nil {
    --- a/bitbucket/converter.go Tue Jan 14 01:08:46 2020 -0600
    +++ b/bitbucket/converter.go Tue Jan 14 04:41:41 2020 -0600
    @@ -77,8 +77,9 @@
    yt.State = replace
    }
    - // add the comments
    + // add the comments and attachments
    yt.Comments = a.convertComments(userMap, bb.ID)
    + yt.Attachments = a.convertAttachments(userMap, bb)
    return yt
    }
    @@ -144,6 +145,34 @@
    return comments
    }
    +func (a *Archive) convertAttachments(userMap map[string]*youtrack.User, issue Issue) []*youtrack.Attachment {
    + attachments := []*youtrack.Attachment{}
    +
    + for _, attachment := range a.Attachments {
    + if attachment.Issue != issue.ID {
    + continue
    + }
    +
    + author := "admin"
    + if user, found := userMap[attachment.User.AccountID]; found {
    + author = user.Login
    + }
    +
    + reader, err := a.OpenAttachment(attachment.Path)
    + if err != nil {
    + fmt.Printf("%v\n", err)
    + continue
    + }
    +
    + yt := youtrack.NewAttachment(attachment.Issue, author, issue.CreatedOn)
    + yt.AddFile(reader, attachment.Filename)
    +
    + attachments = append(attachments, yt)
    + }
    +
    + return attachments
    +}
    +
    func (a *Archive) convert(usersMap *UsersMap) (*youtrack.Project, error) {
    users, err := a.ExtractUsers(usersMap)
    if err != nil {
    --- a/bitbucket/types.go Tue Jan 14 01:08:46 2020 -0600
    +++ b/bitbucket/types.go Tue Jan 14 04:41:41 2020 -0600
    @@ -1,6 +1,7 @@
    package bitbucket
    import (
    + "archive/zip"
    "time"
    )
    @@ -88,4 +89,6 @@
    Components []Component `json:"components"`
    Milestones []Milestone `json:"milestones"`
    Versions []Version `json:"versions"`
    +
    + zip *zip.ReadCloser
    }
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/youtrack/attachment.go Tue Jan 14 04:41:41 2020 -0600
    @@ -0,0 +1,107 @@
    +package youtrack
    +
    +import (
    + "fmt"
    + "io"
    + "mime/multipart"
    + "net/http"
    + "net/url"
    + "strings"
    + "time"
    +)
    +
    +type Attachment struct {
    + issue int
    + author string
    + created time.Time
    + payload *strings.Builder
    +
    + writer *multipart.Writer
    +
    + files []string
    +}
    +
    +func NewAttachment(issue int, author string, created time.Time) *Attachment {
    + a := &Attachment{
    + issue: issue,
    + author: author,
    + created: created,
    + files: []string{},
    + }
    +
    + a.payload = &strings.Builder{}
    + a.writer = multipart.NewWriter(a.payload)
    +
    + return a
    +}
    +
    +func (a *Attachment) Payload() (string, error) {
    + if err := a.writer.Close(); err != nil {
    + return "", err
    + }
    +
    + return a.payload.String(), nil
    +}
    +
    +func (a *Attachment) AddFile(r io.ReadCloser, filename string) error {
    + w, err := a.writer.CreateFormFile(filename, filename)
    + if err != nil {
    + return err
    + }
    +
    + _, err = io.Copy(w, r)
    + if err != nil {
    + return err
    + }
    + defer r.Close()
    +
    + a.files = append(a.files, filename)
    +
    + return nil
    +}
    +
    +func (a *Attachment) ContentType() string {
    + return a.writer.FormDataContentType()
    +}
    +
    +func (c *Client) importAttachment(p *Project, a *Attachment) error {
    + values := url.Values{}
    + values.Add("authorLogin", a.author)
    + values.Add("created", formatTimeString(a.created))
    +
    + payload, err := a.Payload()
    + if err != nil {
    + return err
    + }
    +
    + reader := strings.NewReader(payload)
    + issueID := fmt.Sprintf("%s-%d", p.ID, a.issue)
    +
    + resp, err := c.requestContentType(
    + http.MethodPost,
    + c.uri+"/import/"+issueID+"/attachment?"+values.Encode(),
    + a.ContentType(),
    + reader,
    + )
    + if err != nil {
    + return err
    + }
    + defer resp.Body.Close()
    +
    + return c.validateXmlImportReport(
    + []int{http.StatusOK, http.StatusCreated},
    + resp,
    + )
    +}
    +
    +func (c *Client) ImportAttachments(p *Project) error {
    + for _, issue := range p.Issues {
    + for _, attachment := range issue.Attachments {
    + if err := c.importAttachment(p, attachment); err != nil {
    + return err
    + }
    + }
    + }
    +
    + return nil
    +}
    --- a/youtrack/client.go Tue Jan 14 01:08:46 2020 -0600
    +++ b/youtrack/client.go Tue Jan 14 04:41:41 2020 -0600
    @@ -69,14 +69,17 @@
    return c.client.Do(req)
    }
    -func (c *Client) xmlRequest(method, url string, body io.Reader) (*http.Response, error) {
    +func (c *Client) requestContentType(method, url, contentType string, body io.Reader) (*http.Response, error) {
    req, err := http.NewRequest(method, url, body)
    if err != nil {
    return nil, err
    }
    - req.Header.Add("Content-Type", "application/xml")
    + req.Header.Add("Content-Type", contentType)
    return c.client.Do(req)
    +}
    +func (c *Client) xmlRequest(method, url string, body io.Reader) (*http.Response, error) {
    + return c.requestContentType(method, url, "application/xml", body)
    }
    --- a/youtrack/import.go Tue Jan 14 01:08:46 2020 -0600
    +++ b/youtrack/import.go Tue Jan 14 04:41:41 2020 -0600
    @@ -51,5 +51,10 @@
    return err
    }
    + // create the attachments
    + if err := client.ImportAttachments(p); err != nil {
    + return err
    + }
    +
    return nil
    }
    --- a/youtrack/issue.go Tue Jan 14 01:08:46 2020 -0600
    +++ b/youtrack/issue.go Tue Jan 14 04:41:41 2020 -0600
    @@ -29,7 +29,8 @@
    Type string `yt:"type"`
    State string `yt:"state"`
    - Comments []Comment
    + Comments []Comment
    + Attachments []*Attachment
    }
    func (i *Issue) encode() xmlIssue {
    @@ -125,24 +126,10 @@
    }
    defer resp.Body.Close()
    - if err := c.checkStatuses(resp, []int{http.StatusOK, http.StatusBadRequest}); err != nil {
    - return fmt.Errorf("failed to import issues: %v", err)
    - }
    -
    - report := xmlImportReport{}
    - if err := xml.NewDecoder(resp.Body).Decode(&report); err != nil {
    - return err
    - }
    -
    - if errs := report.Validate(); len(errs) > 0 {
    - for _, err := range errs {
    - fmt.Printf("%v\n", err)
    - }
    -
    - return fmt.Errorf("error importing issues")
    - }
    -
    - return nil
    + return c.validateXmlImportReport(
    + []int{http.StatusOK, http.StatusBadRequest},
    + resp,
    + )
    }
    func (c *Client) ImportIssues(p *Project) error {
    --- a/youtrack/xml.go Tue Jan 14 01:08:46 2020 -0600
    +++ b/youtrack/xml.go Tue Jan 14 04:41:41 2020 -0600
    @@ -3,6 +3,7 @@
    import (
    "encoding/xml"
    "fmt"
    + "net/http"
    "strings"
    "time"
    )
    @@ -69,6 +70,27 @@
    Items []xmlImportReportItem `xml:"item"`
    }
    +func (c *Client) validateXmlImportReport(statusCodes []int, resp *http.Response) error {
    + if err := c.checkStatuses(resp, statusCodes); err != nil {
    + return err
    + }
    +
    + report := xmlImportReport{}
    + if err := xml.NewDecoder(resp.Body).Decode(&report); err != nil {
    + return err
    + }
    +
    + if errs := report.Validate(); len(errs) > 0 {
    + for _, err := range errs {
    + fmt.Printf("%v\n", err)
    + }
    +
    + return fmt.Errorf("failed to import values")
    + }
    +
    + return nil
    +}
    +
    func (x *xmlImportReport) Validate() []error {
    ret := []error{}