Browse Source

experiment

Signed-off-by: Erik Hollensbe <github@hollensbe.org>
main
Erik Hollensbe 8 months ago
commit
b90f5fe0d5
  1. 10
      go.mod
  2. 31
      go.sum
  3. 271
      sink.go

10
go.mod

@ -0,0 +1,10 @@
module code.hollensbe.org/erikh/sink
go 1.15
require (
github.com/emersion/go-imap v1.0.6
github.com/emersion/go-maildir v0.2.0
github.com/urfave/cli/v2 v2.3.0
golang.org/x/text v0.3.5-0.20201125200606-c27b9fd57aec // indirect
)

31
go.sum

@ -0,0 +1,31 @@
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/emersion/go-imap v1.0.6 h1:N9+o5laOGuntStBo+BOgfEB5evPsPD+K5+M0T2dctIc=
github.com/emersion/go-imap v1.0.6/go.mod h1:yKASt+C3ZiDAiCSssxg9caIckWF/JG7ZQTO7GAmvicU=
github.com/emersion/go-maildir v0.2.0 h1:fC4+UVGl8GcQGbFF7AWab2JMf4VbKz+bMNv07xxhzs8=
github.com/emersion/go-maildir v0.2.0/go.mod h1:I2j27lND/SRLgxROe50Vam81MSaqPFvJ0OHNnDZ7n84=
github.com/emersion/go-message v0.11.1/go.mod h1:C4jnca5HOTo4bGN9YdqNQM9sITuT3Y0K6bSUw9RklvY=
github.com/emersion/go-sasl v0.0.0-20191210011802-430746ea8b9b h1:uhWtEWBHgop1rqEk2klKaxPAkVDCXexai6hSuRQ7Nvs=
github.com/emersion/go-sasl v0.0.0-20191210011802-430746ea8b9b/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k=
github.com/emersion/go-textwrapper v0.0.0-20160606182133-d0e65e56babe/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U=
github.com/martinlindhe/base36 v1.0.0/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.5-0.20201125200606-c27b9fd57aec h1:A1qYjneJuzBZZ2gIB8rd6zrfq6l7SoEMJ8EsSilNK/U=
golang.org/x/text v0.3.5-0.20201125200606-c27b9fd57aec/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

271
sink.go

@ -0,0 +1,271 @@
package main
import (
"crypto/tls"
"errors"
"fmt"
"io"
"os"
"sync"
"github.com/emersion/go-imap"
"github.com/emersion/go-imap/client"
"github.com/emersion/go-maildir"
"github.com/urfave/cli/v2"
)
type mailbox struct {
mailbox *imap.MailboxInfo
uids []uint32
}
func main() {
app := cli.NewApp()
app.Authors = []*cli.Author{{Name: "Erik Hollensbe", Email: "github@hollensbe.org"}}
app.Flags = []cli.Flag{
&cli.UintFlag{
Name: "connections",
Usage: "Count of connections to spawn + 1 for listing mailboxes",
Aliases: []string{"c", "conn"},
Value: 5,
},
&cli.UintFlag{
Name: "batch",
Usage: "batch messages by this many",
Aliases: []string{"b"},
Value: 1000,
},
&cli.StringFlag{
Name: "username",
Usage: "Username to login with",
Aliases: []string{"u", "user"},
EnvVars: []string{"SINK_USERNAME"},
},
&cli.StringFlag{
Name: "password",
Usage: "Password to login with",
Aliases: []string{"p", "pass"},
EnvVars: []string{"SINK_PASSWORD"},
},
&cli.StringFlag{
Name: "host",
Usage: "Host to connect to: addr:port",
Aliases: []string{"t"},
EnvVars: []string{"SINK_HOST"},
Value: "imap.gmail.com:993",
},
&cli.BoolFlag{
Name: "noverify",
Aliases: []string{"n"},
Usage: "Don't verify TLS certificates",
},
}
app.Action = run
if err := app.Run(os.Args); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func run(ctx *cli.Context) error {
if ctx.Args().Len() != 1 {
return errors.New("invalid arguments; try --help")
}
tlsConfig := &tls.Config{InsecureSkipVerify: ctx.Bool("noverify")}
conns := ctx.Uint("connections")
batch := ctx.Uint("batch")
host := ctx.String("host")
username := ctx.String("username")
password := ctx.String("password")
var dir maildir.Dir
if os.Getenv("SINK_NO_WRITE") == "" {
dir = maildir.Dir(ctx.Args().First())
if err := dir.Init(); err != nil {
return fmt.Errorf("could not init maikdir: %v", err)
}
if err := dir.Clean(); err != nil {
return err
}
}
c, err := client.DialTLS(host, tlsConfig)
if err != nil {
return fmt.Errorf("While dialing imap server: %v", err)
}
fmt.Println("IMAP connected")
if err := c.Login(username, password); err != nil {
return fmt.Errorf("Failed to login: %v", err)
}
defer c.Logout()
fmt.Println("Logged in")
mailboxes := make(chan *imap.MailboxInfo)
done := make(chan error, 1)
go func() { done <- c.List("", "*", mailboxes) }()
mbs := []*imap.MailboxInfo{}
for mb := range mailboxes {
mbs = append(mbs, mb)
}
if err := <-done; err != nil {
return err
}
mailinfo := make(chan mailbox, conns)
listdone := make(chan error, 1)
messages := make(chan *imap.Message, conns*batch)
allDone := make(chan struct{})
wg := &sync.WaitGroup{}
wg.Add(int(conns))
go func() {
for _, mb := range mbs {
fmt.Println("Spooling", mb.Name)
selected := true
for _, attr := range mb.Attributes {
if attr == imap.NoSelectAttr {
selected = false
break
}
}
if selected {
stat, err := c.Select(mb.Name, true)
if err != nil {
listdone <- fmt.Errorf("Selecting mailbox %q: %v", mb.Name, err)
return
}
if stat.Messages > 0 {
uids, err := c.UidSearch(imap.NewSearchCriteria())
if err != nil {
listdone <- err
return
}
for i := uint(0); i < uint(len(uids)); i += batch {
b := i + batch
if b >= uint(len(uids)) {
b = uint(len(uids)) - 1
}
mailinfo <- mailbox{mailbox: mb, uids: uids[i:b]}
}
}
}
}
close(mailinfo)
wg.Wait()
close(allDone)
}()
for i := uint(0); i < conns; i++ {
go func(wg *sync.WaitGroup) {
defer wg.Done()
c, err := client.DialTLS(ctx.String("host"), tlsConfig)
if err != nil {
listdone <- fmt.Errorf("While dialing imap server: %v", err)
return
}
if err := c.Login(ctx.String("username"), ctx.String("password")); err != nil {
listdone <- fmt.Errorf("Failed to login: %v", err)
return
}
defer c.Logout()
fmt.Println("Subconnection login")
for m := range mailinfo {
name := m.mailbox.Name
fmt.Printf("%p: selecting %q\n", c, name)
if _, err := c.Select(name, true); err != nil {
listdone <- err
return
}
for offset := uint(0); offset < uint(len(m.uids)); offset += batch {
set := &imap.SeqSet{}
for i := offset; i < offset+batch; i++ {
if uint(len(m.uids)) <= i {
break
}
set.AddNum(m.uids[i])
}
msgs := make(chan *imap.Message, batch)
done := make(chan error, 1)
section := imap.BodySectionName{Peek: true}
go func() {
done <- c.UidFetch(set, []imap.FetchItem{imap.FetchEnvelope, section.FetchItem()}, msgs)
}()
if err := <-done; err != nil {
listdone <- fmt.Errorf("%q: UidSearch response: %v: %v", name, err, set)
return
}
for msg := range msgs {
if os.Getenv("SINK_NO_WRITE") == "" {
r := msg.GetBody(&section)
_, w, err := dir.Create(nil)
if err != nil {
listdone <- fmt.Errorf("%q: error creating entry in maildir: %v", name, err)
return
}
if _, err := io.Copy(w, r); err != nil {
listdone <- fmt.Errorf("%q: failure writing mail entry: %v", name, err)
return
}
w.Close()
}
messages <- msg
}
}
}
}(wg)
}
count := 0
for {
select {
case err := <-listdone:
return err
case <-allDone:
return nil
case m := <-messages:
from := "No From Address"
if len(m.Envelope.From) > 0 {
from = m.Envelope.From[0].Address()
}
fmt.Printf("Message #%d: %s %s\n", count, from, m.Envelope.Subject)
count++
}
}
}
Loading…
Cancel
Save