This documentation is for Dovecot v1.x, see wiki2 for v2.x documentation.

Attachment 'uw2dovecot.pl'

Download

   1 #!/usr/bin/perl -w
   2 
   3 # -i inbox [INBOX]
   4 # -u uwmaildir [mail] ('' to skip)
   5 # -s subscriptions [.mailboxlist] ('' to skip)
   6 # -m maildir [Maildir]
   7 #
   8 # maildir must not exist
   9 
  10 use strict;
  11 use Getopt::Std;
  12 use Time::Local;
  13 use File::Find;
  14 use File::Path;
  15 use File::Basename;
  16 
  17 my %opts;
  18 my $inbox;
  19 my $uwmaildir;
  20 my $subscriptions;
  21 my $maildir;
  22 my @inbox_stat;
  23 my @maildir_stat;
  24 my $hostname;
  25 my $totalmsgs = 0;
  26 my %months = (
  27 	'Jan' => 0,
  28 	'Feb' => 1,
  29 	'Mar' => 2,
  30 	'Apr' => 3,
  31 	'May' => 4,
  32 	'Jun' => 5,
  33 	'Jul' => 6,
  34 	'Aug' => 7,
  35 	'Sep' => 8,
  36 	'Oct' => 9,
  37 	'Nov' => 10,
  38 	'Dec' => 11
  39 );
  40 
  41 # mix format:
  42 #	.mixmeta:
  43 #		V UIDVALIDITY(hex)
  44 #		K key0 key1 key2
  45 #	.mixindex:
  46 #		:uid:yyyymmddhhmmss[-+]zzzz:rfcsize:fileid:offset:mixhdrsz:hdrsz:
  47 #	.mixstatus:
  48 #		:uid:keywordflags:sysflags:modval:
  49 #		keywordflags from .mixmeta
  50 #		sysflags:
  51 #			0x01 = SEEN
  52 #			0x02 = TRASH
  53 #			0x04 = FLAG
  54 #			0x08 = REPLIED
  55 #			0x10 = OLD (cur, else new)
  56 #			0x20 = DRAFT (not used)
  57 #
  58 # mbx format:
  59 #	all data has \r\n
  60 #	*mbx*
  61 #	VVVVVVVVUUUUUUUU (V=VALIDITY, U=NEXT UID but often lazy assignment)
  62 #	key0
  63 #	key1
  64 #	...
  65 #	key29
  66 #	pad to 2048 bytes, last 10 bytes can be LASTPID\r\n
  67 #	DD-MMM-YYYY HH:MM:SS [+-]ZZZZ,length(dec);kkkkkkkkssss-uuuuuuuu
  68 #	msg
  69 #	...
  70 #
  71 # mbox format:
  72 #	From_ lines at begining of each message
  73 #	can have:
  74 #	X-IMAP: UIDVALIDITY MEXTUID (often lazy assignment)
  75 #	X-IMAPbase: UIDVALIDITY NEXTUID (lazy and indicates pseudo message)
  76 #	Status: FLAGS
  77 #	X-Status: FLAGS
  78 #	X-Keywords: key ...
  79 #
  80 # .mailboxlist:
  81 #	folder (strip mail/, convert / to .)
  82 #	...
  83 #
  84 # dovecot-keywords:
  85 #	number(dec) keyword
  86 #	...
  87 # dovecot-uidlist:
  88 #	1 UIDVALIDITY(dec) NEXT
  89 #	uid(dec) filename
  90 # subscriptions
  91 #	folder
  92 #	...
  93 # cur
  94 #	timestamp.uid.hostname:2,flags
  95 #	flags = F(lag), R(eplied), S(een), T(rash), a-z (keywords)
  96 # new
  97 #	timestamp.uid.hostname
  98 # .sub.folder:
  99 #	maildirfolder (exists to make dovecot happy)
 100 #	dovecot-keywords (as above)
 101 #	dovecot-uidlist (as above)
 102 
 103 sub convert($$$) {
 104 	my $mailbox = shift(@_);
 105 	my $outdir = shift(@_);
 106 	my $subfolder = shift(@_);
 107 	my $uidvalidity;
 108 	my @keywords;
 109 	my $line;
 110 	my %msgs;
 111 	if (-d $mailbox) {
 112 		eval {
 113 			open(META, '<', "$mailbox/.mixmeta") || die "Can't open $mailbox/.mixmeta";
 114 			open(INDEX, '<', "$mailbox/.mixindex") || die "Can't open $mailbox/.mixindex";
 115 			open(STATUS, '<', "$mailbox/.mixstatus") || die "Can't open $mailbox/.mixstatus";
 116 		};
 117 		if ($@) {
 118 			warn $@;
 119 			return;
 120 		}
 121 		while ($line = <META>) {
 122 			if ($line =~ m/^V([[:xdigit:]]{8})\r\n$/) {
 123 				$uidvalidity = hex($1);
 124 			} elsif ($line =~ m/^K(.*)\r\n$/) {
 125 				@keywords = split(' ', $1);
 126 			}
 127 		}
 128 		close(META);
 129 		if (!defined($uidvalidity)) {
 130 			warn "$mailbox: No uidvalidity";
 131 			return;
 132 		}
 133 		while ($line = <INDEX>) {
 134 			if ($line =~ m/^:([[:xdigit:]]{8}):(\d\d\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)([-+])(\d\d)(\d\d):([[:xdigit:]]{8}):([[:xdigit:]]{8}):([[:xdigit:]]{8}):([[:xdigit:]]{8}):([[:xdigit:]]{8}):\r\n$/) {
 135 				$msgs{$1} = {
 136 					'timestamp' => timegm($7, $6, $5, $4, $3-1, $2) + (($8 eq '-' ? 1 : -1)*($9 * 60 + $10)*60),
 137 					'size' => hex($11),
 138 					'filename' => "$mailbox/.mix$12",
 139 					'offset' => hex($13),
 140 					'skip' => hex($14),
 141 				};
 142 			}
 143 		}
 144 		close(INDEX);
 145 		while ($line = <STATUS>) {
 146 			if ($line =~ m/^:([[:xdigit:]]{8}):([[:xdigit:]]{8}):([[:xdigit:]]{4}):([[:xdigit:]]{8}):\r\n$/) {
 147 				$msgs{$1}{'new'} = !(hex($3) & 0x10);
 148 				$msgs{$1}{'flags'} =
 149 					((hex($3) & 0x1) ? 'S' : '').
 150 					((hex($3) & 0x2) ? 'T' : '').
 151 					((hex($3) & 0x4) ? 'F' : '').
 152 					((hex($3) & 0x8) ? 'R' : '');
 153 				foreach my $i (0 .. 25) {
 154 					if (hex($2) & (1 << $i)) {
 155 						$msgs{$1}{'flags'} .= chr(ord('a') + $i);
 156 					}
 157 				}
 158 			}
 159 		}
 160 		close(STATUS);
 161 		$totalmsgs += scalar(keys(%msgs));
 162 		print "$mailbox: Converting mix file (".scalar(keys(%msgs))." messages)\n" if $opts{'v'};
 163 	} else {
 164 		close(MAILBOX); #force $. reset
 165 		open(MAILBOX, '<', $mailbox);
 166 		$line = <MAILBOX>;
 167 		if (!defined($line)) {
 168 			print "$mailbox: Empty file (0 messages)\n" if $opts{'v'};
 169 		} elsif ($line eq "*mbx*\r\n") {
 170 			$line = <MAILBOX>;
 171 			if ($line =~ /^([[:xdigit:]]{8})([[:xdigit:]]{8})\r\n$/) {
 172 				$uidvalidity = hex($1);
 173 			} else {
 174 				warn "$mailbox: Bogus UID line";
 175 				return;
 176 			}
 177 			foreach my $n (0 .. 29) {
 178 				$line = <MAILBOX>;
 179 				$line =~ s/\r\n//;
 180 				if ($line ne '') {
 181 					push(@keywords, $line);
 182 				}
 183 			}
 184 			seek(MAILBOX, 2048, 0);
 185 			my $lazyuid = 0;
 186 			while ($line = <MAILBOX>) {
 187 				if ($line =~ m/( \d|\d\d)-(\w\w\w)-(\d\d\d\d) (\d\d):(\d\d):(\d\d) ([+-])(\d\d)(\d\d),(\d+);([[:xdigit:]]{8})([[:xdigit:]]{4})-([[:xdigit:]]{8})\r\n$/) {
 188 					if ($13 eq '00000000') {
 189 						$lazyuid++;
 190 					} else {
 191 						$lazyuid = hex($13);
 192 					}
 193 					my $hexuid = sprintf('%08x', $lazyuid);
 194 					$msgs{$hexuid} = {
 195 						'timestamp' => timegm($6, $5, $4, $1+0, $months{$2}, $3) + (($8 eq '-' ? 1 : -1)*($8 * 60 + $9)*60),
 196 						'size' => $10,
 197 						'filename' => $mailbox,
 198 						'offset' => tell(MAILBOX),
 199 						'skip' => 0,
 200 						'new' => !(hex($12) & 0x10),
 201 						'flags' =>
 202 							((hex($12) & 0x1) ? 'S' : '').
 203 							((hex($12) & 0x2) ? 'T' : '').
 204 							((hex($12) & 0x4) ? 'F' : '').
 205 							((hex($12) & 0x8) ? 'R' : ''),
 206 					};
 207 					foreach my $i (0 .. 25) {
 208 						if (hex($11) & (1 << $i)) {
 209 							$msgs{$hexuid}{'flags'} .= chr(ord('a') + $i);
 210 						}
 211 					}
 212 					seek(MAILBOX, $msgs{$hexuid}{'size'}, 1);
 213 				} else {
 214 					warn "Bogus line in mbx";
 215 				}
 216 			}
 217 			$totalmsgs += scalar(keys(%msgs));
 218 			print "$mailbox: Converting mbx file (".scalar(keys(%msgs))." messages)\n" if $opts{'v'};
 219 		} elsif ($line =~ m/^From /) {
 220 			seek(MAILBOX, 0, 0);
 221 			my $lazyuid = 0;
 222 			my $tmpoffset;
 223 			my $tmptimestamp;
 224 			my %tmpkeywords = ();
 225 			my $tmpflags;
 226 			my $tmpnew;
 227 			my $pseudomsg;
 228 			my $end = 0;
 229 			my $inheader = 0;
 230 			$uidvalidity = time();
 231 			while ($line = <MAILBOX>) {
 232 				if ($line =~ m/^From (?:\S+)\s+... ... (?: \d|\d\d) \d\d:\d\d:\d\d \d\d\d\d(?: [+-]\d\d\d\d)?\n$/) {
 233 					if ($end > 0 && !$pseudomsg) {
 234 						# found end of current message, capture info
 235 						my $hexuid = sprintf('%08x', $lazyuid);
 236 						$msgs{$hexuid} = {
 237 							'timestamp' => $tmptimestamp,
 238 							'size' => $end - $tmpoffset,
 239 							'filename' => $mailbox,
 240 							'offset' => $tmpoffset,
 241 							'skip' => 0,
 242 							'new' => $tmpnew,
 243 							'flags' => $tmpflags
 244 						};
 245 					}
 246 					# capture $n vars here, just to avoid confusion, but to confuse, (?:) is grouping without capturing
 247 					$line =~ m/^From (?:\S+)\s+... (...) ( \d|\d\d) (\d\d):(\d\d):(\d\d) (\d\d\d\d)(?: ([+-])(\d\d)(\d\d))?\n$/;
 248 					$tmpoffset = tell(MAILBOX);
 249 					$inheader = 1;
 250 					$tmptimestamp = timegm($5, $4, $3, $2+0, $months{$1}, $6) + (defined($8) ? (($7 eq '-' ? 1 : -1)*($8 * 60 + $9)*60) : 0);
 251 					$tmpflags = '';
 252 					$tmpnew = 1;
 253 					$lazyuid++;
 254 					$pseudomsg = 0;
 255 				} elsif ($inheader) {
 256 					if ($line =~ m/X-IMAP(base)?: (\d+) (\d+)\n$/) {
 257 						$uidvalidity = $2;
 258 						# X-IMAP: means pseudo message
 259 						if (!defined($1)) {
 260 							$pseudomsg = 1;
 261 							$lazyuid = 0;
 262 						}
 263 					} elsif ($line =~ m/X-Keywords:\s+(.*)\n$/) {
 264 						foreach my $kw (split(' ', $1)) {
 265 							if (!defined($tmpkeywords{$kw})) {
 266 								$tmpkeywords{$kw} = scalar(@keywords);
 267 								push(@keywords, $kw);
 268 							}
 269 							if ($tmpkeywords{$kw} < 26) {
 270 								$tmpflags .= chr(ord('a') + $tmpkeywords{$kw});
 271 							}
 272 						}
 273 					} elsif ($line =~ m/X-UID: (\d+)\n$/) {
 274 						$lazyuid = $1;
 275 					} elsif ($line =~ m/^(X-)?Status: (\S+)/) {
 276 						foreach my $f (split(//, $2)) {
 277 							if ($f eq 'R') {
 278 								$tmpflags .= 'S';
 279 							} elsif ($f eq 'A') {
 280 								$tmpflags .= 'R';
 281 							} elsif ($f eq 'F') {
 282 								$tmpflags .= 'F';
 283 							} elsif ($f eq 'D') {
 284 								$tmpflags .= 'T';
 285 							} elsif ($f eq 'O') {
 286 								$tmpnew = 0;
 287 							}
 288 						}
 289 					} elsif ($line =~ m/^\n$/) {
 290 						$inheader = 0;
 291 						$end = tell(MAILBOX);
 292 					}
 293 				} else {
 294 					$end = tell(MAILBOX);
 295 				}
 296 			}
 297 			# catch last message (if one)
 298 			if ($end > 0 && !$pseudomsg) {
 299 				# found end of current message, capture info
 300 				my $hexuid = sprintf('%08x', $lazyuid);
 301 				$msgs{$hexuid} = {
 302 					'timestamp' => $tmptimestamp,
 303 					'size' => $end - $tmpoffset,
 304 					'filename' => $mailbox,
 305 					'offset' => $tmpoffset,
 306 					'skip' => 0,
 307 					'new' => $tmpnew,
 308 					'flags' => $tmpflags
 309 				};
 310 			}
 311 			$totalmsgs += scalar(keys(%msgs));
 312 			print "$mailbox: Converting mbox file (".scalar(keys(%msgs))." messages)\n" if $opts{'v'};
 313 		} else {
 314 			print "$mailbox: Unknown file format, skipping\n" if $opts{'v'};
 315 			return;
 316 		}
 317 	}
 318 	eval {
 319 		mkpath($outdir);
 320 	};
 321 	if ($@) {
 322 		warn $@;
 323 		return;
 324 	}
 325 	if (scalar(@keywords) > 26) {
 326 		warn "$mailbox: Too many keywords, only first 26 will be kept";
 327 		@keywords=@keywords[0 .. 25];
 328 	}
 329 	if (scalar(@keywords) > 0) {
 330 		open(KEYWORDS, '>', "$outdir/dovecot-keywords");
 331 		foreach my $kn (0 .. $#keywords) {
 332 			print KEYWORDS "$kn ${keywords[$kn]}\n";
 333 		}
 334 		close(KEYWORDS);
 335 	}
 336 	mkdir("$outdir/tmp");
 337 	mkdir("$outdir/new");
 338 	mkdir("$outdir/cur");
 339 	if ($subfolder) {
 340 		open(SUBFOLDER, '>', "$outdir/maildirfolder");
 341 		close(SUBFOLDER);
 342 	}
 343 	if (scalar(keys(%msgs))) {
 344 		my $maxuidl = 0;
 345 		foreach my $uidl (sort(keys(%msgs))) {
 346 			if (hex($uidl) > $maxuidl) {
 347 				$maxuidl = hex($uidl);
 348 			}
 349 		}
 350 		open(UIDLIST, '>', "$outdir/dovecot-uidlist");
 351 		print UIDLIST "1 $uidvalidity ".($maxuidl + 1)."\n";
 352 		foreach my $uidl (sort(keys(%msgs))) {
 353 			my $msg = $msgs{$uidl};
 354 			my $data;
 355 			eval {
 356 				open(MSG, '<', $msg->{'filename'}) || die "Can't open ".$msg->{'filename'};
 357 				seek(MSG, $msg->{'offset'}+$msg->{'skip'}, 0) || die "Can't seek to ".($msg->{'offset'}+$msg->{'skip'});
 358 				read(MSG, $data, $msg->{'size'}) || die "Can't read ".$msg->{'size'}." bytes";
 359 				close(MSG);
 360 			};
 361 			if ($@) {
 362 				warn $@;
 363 				next;
 364 			}
 365 			$data =~ s/\r\n/\n/g;
 366 			my $filebase = $msg->{'timestamp'}.'.'.$uidl.'.'.$hostname;
 367 			if (!$msg->{'new'}) {
 368 				$filebase .= ':2,'.$msg->{'flags'};
 369 			}
 370 			my $filename = $outdir.'/'.($msg->{'new'} ? 'new' : 'cur').'/'.$filebase;
 371 			print UIDLIST hex($uidl)." $filebase\n";
 372 			open(NEWFILE, '>', $filename);
 373 			print NEWFILE $data;
 374 			close(NEWFILE);
 375 			utime($msg->{'timestamp'}, $msg->{'timestamp'}, $filename);
 376 		}
 377 		close(UIDLIST);
 378 	}
 379 }
 380 
 381 sub findfunc() {
 382 	my @s = stat($_);
 383 	# check for inbox or maildir and prune
 384 	if (($s[1] == $inbox_stat[1] && $s[0] == $inbox_stat[0]) || ($s[1] == $maildir_stat[1] && $s[0] == $maildir_stat[0])) {
 385 		$File::Find::prune = 1;
 386 		return;
 387 	}
 388 	if (basename($_) =~ m/^\.mix/) {
 389 		return;
 390 	}
 391 	if (-d $_ && ! -e $_.'/.mixstatus') {
 392 		return;
 393 	}
 394 	my $tmpnam = $File::Find::name;
 395 	$tmpnam =~ s/^\Q$uwmaildir\E\///;
 396 	$tmpnam =~ s/\./_/g;
 397 	$tmpnam =~ s/\//./g;
 398 	convert($File::Find::name, $maildir.'/.'.$tmpnam, 1);
 399 }
 400 
 401 #
 402 # main body
 403 #
 404 
 405 getopts('i:u:s:m:v', \%opts);
 406 $inbox = defined($opts{'i'}) ? $opts{'i'} : 'INBOX';
 407 $uwmaildir = defined($opts{'u'}) ? $opts{'u'} : 'mail';
 408 $subscriptions = defined($opts{'s'}) ? $opts{'s'} : '.mailboxlist';
 409 $maildir = defined($opts{'m'}) ? $opts{'m'} : 'Maildir';
 410 chomp($hostname = `hostname`);
 411 
 412 die "$maildir must not exist" if (-e $maildir);
 413 die "$inbox doesn't exist" if (! -e $inbox);
 414 die "$uwmaildir doesn't exist" if ($uwmaildir ne '' && ! -d $uwmaildir);
 415 die "$subscriptions doesn't exist" if ($subscriptions ne '' && ! -e $subscriptions);
 416 
 417 umask(077);
 418 
 419 convert($inbox, $maildir, 0);
 420 
 421 # get dev/ino info for inbox and maildir so if we happen to be converting . we don't convert them as well
 422 @inbox_stat = stat($inbox);
 423 @maildir_stat = stat($maildir);
 424 
 425 if ($uwmaildir ne '') {
 426 	find({ 'wanted' => \&findfunc, 'no_chdir' => 1 }, $uwmaildir);
 427 }
 428 
 429 if ($subscriptions ne '') {
 430 	if (open(SUBS, '<', $subscriptions)) {
 431 		eval {
 432 			open(NEWSUBS, '>', "$maildir/subscriptions") || die "Can't open $maildir/subscriptions";
 433 		};
 434 		if ($@) {
 435 			warn $@;
 436 		} else {
 437 			my $line;
 438 			while ($line = <SUBS>) {
 439 				$line =~ s/^\Q$uwmaildir\E\///;
 440 				$line =~ s/\./_/g;
 441 				$line =~ s/\//./g;
 442 				print NEWSUBS $line;
 443 			}
 444 		}
 445 		close(SUBS);
 446 		close(NEWSUBS);
 447 	} else {
 448 		warn "Can't open $subscriptions"
 449 	}
 450 }
 451 
 452 print "Total conversion: $totalmsgs messages\n" if ($opts{'v'});

Attached Files

To refer to attachments on a page, use attachment:filename, as shown below in the list of files. Do NOT use the URL of the [get] link, since this is subject to change and can break easily.
  • [get | view] (2017-03-28 09:31:08, 11.8 KB) [[attachment:uw2dovecot.pl]]
 All files | Selected Files: delete move to page copy to page

You are not allowed to attach a file to this page.