1 | #!/usr/bin/perl |
---|
2 | use DBI qw(:sql_types); |
---|
3 | use Cwd; |
---|
4 | $| = 1; |
---|
5 | |
---|
6 | # We want to trap control-c to ensure a clean shutdown |
---|
7 | $SIG{'INT'} = 'shutdown'; |
---|
8 | |
---|
9 | my $storenumber = 0; |
---|
10 | my $store; |
---|
11 | my $lookup; |
---|
12 | my $here = getcwd; |
---|
13 | |
---|
14 | # Set maximum size for a database file |
---|
15 | my $max_size = 3865470566; |
---|
16 | |
---|
17 | |
---|
18 | # Connect to both database files and create the schemas (if necessary) |
---|
19 | my $needcreate = 0; |
---|
20 | unless(-f "lookup.sqlite") |
---|
21 | { |
---|
22 | $needcreate = 1; |
---|
23 | } |
---|
24 | $store = DBI->connect("dbi:SQLite:dbname=store.sqlite.0","","",{AutoCommit => 0}) or die($!); |
---|
25 | $lookup = DBI->connect("dbi:SQLite:dbname=lookup.sqlite","","",{AutoCommit => 0}) or die($!); |
---|
26 | if($needcreate) |
---|
27 | { |
---|
28 | $lookup->do("CREATE TABLE tiles (z integer, x integer, y integer, store_filename string, extension varchar(10), unix_epoch_timestamp integer, primary key (z, x, y, extension))"); |
---|
29 | $lookup->do("CREATE TABLE version (v integer)"); |
---|
30 | $lookup->do("insert into version values(1)"); |
---|
31 | create_store_schema(); |
---|
32 | } |
---|
33 | my $storeinsert = prepare_query(); |
---|
34 | my $lookupinsert = $lookup->prepare("insert into tiles values(?,?,?,?,?,?)"); |
---|
35 | my $check_exists = $lookup->prepare("select store_filename from tiles where z=? and x=? and y=?"); |
---|
36 | |
---|
37 | # Get a list of zoom levels |
---|
38 | my @zooms = <[0-9]*>; |
---|
39 | |
---|
40 | # Work through each zoom level |
---|
41 | |
---|
42 | my $filecount = `find [0-9]* -type f|wc -l`; |
---|
43 | chomp($filecount); |
---|
44 | my $filenum = 0; |
---|
45 | my $transaction = 0; |
---|
46 | my ($z,$x,@x,$y,$file,@files,$files,@data,$data,$stamp,$extension,$existingstore); |
---|
47 | foreach $z(@zooms) |
---|
48 | { |
---|
49 | chdir($z); |
---|
50 | # Get a list of x co-ordinates |
---|
51 | @x = <*>; |
---|
52 | foreach $x(@x) |
---|
53 | { |
---|
54 | chdir($x); |
---|
55 | # Get a list of files |
---|
56 | @files = <*>; |
---|
57 | # Work through the files and add them to the database |
---|
58 | foreach $file(@files) |
---|
59 | { |
---|
60 | chomp($file); |
---|
61 | $filenum++; |
---|
62 | # Get timestamp |
---|
63 | $stamp = (stat($file))[9]; |
---|
64 | # Get file data |
---|
65 | $data = `cat $file`; |
---|
66 | # get extension and filename |
---|
67 | ($y,$extension) = split(/\./,$file); |
---|
68 | # Check if the tile already exists |
---|
69 | $check_exists->execute($z,$x,$y); |
---|
70 | $existingstore = $check_exists->fetchrow_array(); |
---|
71 | if($existingstore eq "") |
---|
72 | { |
---|
73 | # Push into databases |
---|
74 | $storeinsert->bind_param(4,$data,SQL_BLOB); |
---|
75 | $storeinsert->bind_param(1,$z,SQL_INTEGER); |
---|
76 | $storeinsert->bind_param(2,$x,SQL_INTEGER); |
---|
77 | $storeinsert->bind_param(3,$y,SQL_INTEGER); |
---|
78 | $storeinsert->bind_param(5,$extension,SQL_VARCHAR); |
---|
79 | $storeinsert->bind_param(6,$stamp,SQL_INTEGER); |
---|
80 | $storeinsert->execute(); |
---|
81 | $lookupinsert->execute($z,$x,$y,'store.sqlite.' . $storenumber,$extension,$stamp); |
---|
82 | # Only commit every 50 transactions to save time |
---|
83 | $transaction++; |
---|
84 | if($transaction == 50) |
---|
85 | { |
---|
86 | print(" Processed $filenum / $filecount tiles\r"); |
---|
87 | flush(); |
---|
88 | |
---|
89 | # Check the store filesize |
---|
90 | my $size = (stat("$here/store.sqlite.$storenumber"))[7]; |
---|
91 | if($size > $max_size) |
---|
92 | { |
---|
93 | # Need to open a new store file |
---|
94 | $store->disconnect(); |
---|
95 | $storenumber++; |
---|
96 | $store = DBI->connect("dbi:SQLite:dbname=$here/store.sqlite.$storenumber","","",{AutoCommit => 0}) or die($!); |
---|
97 | create_store_schema(); |
---|
98 | $storeinsert = prepare_query(); |
---|
99 | } |
---|
100 | } |
---|
101 | } |
---|
102 | } |
---|
103 | chdir(".."); |
---|
104 | } |
---|
105 | chdir(".."); |
---|
106 | } |
---|
107 | # We're done, so shut down tidily |
---|
108 | flush(); |
---|
109 | $store->disconnect(); |
---|
110 | $lookup->disconnect(); |
---|
111 | |
---|
112 | |
---|
113 | sub prepare_query |
---|
114 | { |
---|
115 | my $storeinsert = $store->prepare("insert into tiles values(?,?,?,?,?,?)"); |
---|
116 | return($storeinsert); |
---|
117 | } |
---|
118 | sub create_store_schema |
---|
119 | { |
---|
120 | $store->do("CREATE TABLE tiles (z integer, x integer, y integer, tile blob, extension varchar(10), unix_epoch_timestamp integer, primary key (z, x, y, extension))"); |
---|
121 | $store->do("CREATE TABLE version (v integer)"); |
---|
122 | $store->do("insert into version values(1)"); |
---|
123 | } |
---|
124 | sub shutdown |
---|
125 | { |
---|
126 | # called when Ctrl-C is hit |
---|
127 | |
---|
128 | # Make sure another click doesn't abort us |
---|
129 | $SIG{'INT'} = 'IGNORE'; |
---|
130 | print("\nShutting down\n"); |
---|
131 | flush(); |
---|
132 | $store->disconnect(); |
---|
133 | $lookup->disconnect(); |
---|
134 | exit; |
---|
135 | } |
---|
136 | sub flush |
---|
137 | { |
---|
138 | # Commits the database. |
---|
139 | $store->commit(); |
---|
140 | $lookup->commit(); |
---|
141 | $transaction = 0; |
---|
142 | } |
---|