Source code: jreceiver/server/scanner/ScannerDaemon.java
1 /* $Header: /cvsroot/jreceiver/jreceiver/src/jreceiver/server/scanner/ScannerDaemon.java,v 1.23 2003/05/16 08:40:01 reedesau Exp $ */
2
3 package jreceiver.server.scanner;
4
5 import java.io.File;
6 import java.io.FileNotFoundException;
7 import java.io.FileFilter;
8 import java.io.IOException;
9 import java.util.Hashtable;
10 import java.util.Iterator;
11 import java.util.List;
12 import java.util.Map;
13 import java.util.Vector;
14
15 import org.apache.commons.logging.Log;
16 import org.apache.commons.logging.LogFactory;
17
18 import jreceiver.common.rec.site.Folder;
19 import jreceiver.common.rec.site.Root;
20 import jreceiver.common.rec.site.RootKey;
21 import jreceiver.common.rec.site.Site;
22 import jreceiver.common.rec.source.Source;
23 import jreceiver.common.rec.source.Mfile;
24 import jreceiver.common.rec.source.Playlist;
25 import jreceiver.common.rpc.Scanner;
26 import jreceiver.common.rpc.RpcException;
27 import jreceiver.server.ScannerSettingCache;
28 import jreceiver.server.bus.BusException;
29 import jreceiver.server.bus.FolderBus;
30 import jreceiver.server.bus.MfileBus;
31 import jreceiver.server.bus.MimeBus;
32 import jreceiver.server.bus.PlaylistBus;
33 import jreceiver.server.bus.RootBus;
34 import jreceiver.server.bus.SourceBus;
35 import jreceiver.server.bus.TuneBus;
36 import jreceiver.server.util.builder.BuilderException;
37 import jreceiver.server.util.builder.SourceBuilder;
38 import jreceiver.util.ScheduledDaemon;
39
40 /**
41 * Daemon to scan the root folders for audio files and update the
42 * database. Report status to listeners.
43 *
44 * synchronize database with audio files on disk
45 * <p>
46 * For each dir in path:<br>
47 * <li>get list of files from db
48 * <li>get list of files from disk
49 * <li>update records where file exists and where date is newer
50 * <li>delete records where file is missing
51 * <li>add records for new files
52 * <p>
53 * Runs as a background worker thread.
54 * <p>
55 * Provides progress messages via XML-RPC.
56 * <p>
57 * @author Reed Esau
58 * @version $Revision: 1.23 $ $Date: 2003/05/16 08:40:01 $
59 */
60 public final class ScannerDaemon extends ScheduledDaemon {
61
62 /**
63 * Obtain the instance of the ScannerDaemon singleton
64 * <p>
65 * Note that this uses the questionable DCL pattern (search on
66 * DoubleCheckedLockingIsBroken for more info)
67 * <p>
68 * Note that the instance method must be called once with
69 * a valid status port before the singleton is created.
70 */
71 public static ScannerDaemon getInstance() {
72 if (singleton == null) {
73 synchronized (ScannerDaemon.class) {
74 if (singleton == null) {
75 singleton = new ScannerDaemon();
76 }
77 }
78 }
79 return singleton;
80 }
81
82
83 /**
84 * ctor
85 */
86 ScannerDaemon() {
87 super("ScannerDaemon");
88
89 m_source_builder = null;
90
91 m_fldr_bus = FolderBus .getInstance();
92 m_mfile_bus = MfileBus .getInstance();
93 m_mime_bus = MimeBus .getInstance();
94 m_pl_bus = PlaylistBus.getInstance();
95 m_root_bus = RootBus .getInstance();
96 m_src_bus = SourceBus .getInstance();
97 m_tune_bus = TuneBus .getInstance();
98
99 m_tune_filter = new TuneFileFilter();
100 m_playlist_filter = new PlaylistFileFilter();
101 m_folder_filter = new FolderFileFilter();
102
103 state = Scanner.SCANNER_STATE_STOPPED;
104 }
105
106 /**
107 * obtain the current state of the scanner
108 *
109 * @return
110 */
111 public synchronized String getState() {
112 return state;
113 }
114
115 //
116 // internal routines
117 //
118
119 /**
120 * the work done in the worker thread
121 * <p>
122 * search for tunes starting at the specified root and add them
123 * to the database
124 * <p>
125 * examine a file or recurse a directory, adding tunes and playlists
126 * and some of their metadata to the database
127 * <p>
128 * set up a callback to enumerate all root folders and scan each one.
129 */
130 protected void doWork() {
131 long start = System.currentTimeMillis();
132 try {
133 log.info("scan start");
134
135 state = Scanner.SCANNER_STATE_STARTED;
136
137 // initialize the builder, if necessary
138 if (m_source_builder == null)
139 m_source_builder = new SourceBuilder();
140 else
141 m_source_builder.resetData();
142
143 scanForMode(Source.SRCTYPE_TUNE);
144 if (stopping()) {
145 log.debug("scan exit after tune scan");
146 return;
147 }
148
149 scanForMode(Source.SRCTYPE_PLAYLIST);
150 if (stopping()) {
151 log.debug("scan exit after playlist scan");
152 return;
153 }
154
155 refreshPlaylistStats();
156 if (stopping()) {
157 log.debug("scan exit after refreshPlaylistStats");
158 return;
159 }
160
161 // trim all dangling references en masse.
162 m_tune_bus.bulkCleanReferences();
163
164 log.info("scan complete (was not interrupted)");
165 } catch (BusException e) {
166 log.error("bus-problem with scanner daemon", e);
167 } catch (Throwable t) {
168 log.error("generic-problem with scanner daemon", t);
169 } finally {
170 long diff = System.currentTimeMillis() - start;
171 log.info("scan complete, timer=" + diff);
172 state = Scanner.SCANNER_STATE_STOPPED;
173 m_source_builder.resetData();
174 }
175 }
176
177
178 //
179 // internal routines
180 //
181
182 /**
183 * Scan for tunes or playlists.
184 *
185 * @param src_type SRCTYPE_TUNE or SRCTYPE_PLAYLIST
186 */
187 void scanForMode(int src_type) throws IOException, BusException {
188 List root_keys = m_root_bus.getKeys(null, //unfiltered
189 null, //default sort order
190 0, Root.NO_LIMIT);
191 if (log.isDebugEnabled())
192 log.debug("scanForMode: src_type=" + src_type + " roots=" + root_keys);
193
194 Iterator it = root_keys.iterator();
195 while (it.hasNext()) {
196 RootKey key = (RootKey)it.next();
197
198 scanCurrent(key.getFilePath(), null/*current*/, src_type);
199 if (stopping()) {
200 log.debug("scanForMode: exit after root scan");
201 return;
202 }
203 }
204 }
205
206
207 /**
208 * search for tunes in the specified folder and add them to the database
209 * <p>
210 * examine a file or recurse a directory, adding tunes and playlists
211 * and some of their metadata to the database
212 */
213 void scanCurrent(File root_folder, File current_folder, int src_type)
214 throws BusException, IOException {
215
216 if (log.isDebugEnabled())
217 log.debug("scan: " + current_folder);
218
219 if (root_folder == null)
220 throw new IllegalArgumentException();
221
222 if (stopping()) {
223 log.debug("scan: stopping early");
224 return;
225 }
226
227 if (current_folder == null) {
228 if (log.isDebugEnabled())
229 log.debug("scan: beginning at root " + root_folder);
230 current_folder = root_folder;
231 }
232
233
234 // Build up database until we have a valid folder id for the given
235 // root and (relative?) path.
236 int folder_id = m_fldr_bus.getFolderId(Site.SITE_HOME, root_folder, current_folder);
237
238 scanFoldersForCurrent(root_folder, current_folder, folder_id, src_type);
239
240 scanFilesForCurrent(current_folder, folder_id, src_type);
241 }
242
243
244 /** scan subfolders for the current folder */
245 void scanFoldersForCurrent(File root_folder,
246 File current_folder,
247 int folder_id,
248 int src_type) throws IOException, BusException {
249
250 // Obtain a list of subfolders which are already referenced
251 // in the database for the current folder. This list will be culled as
252 // we find the physical subfolders. Any references that
253 // remain after the scan can be considered obsolete zombies which can be
254 // deleted.
255 Map referenced = m_fldr_bus.getChildMap(folder_id, 0, Folder.NO_LIMIT);
256
257 File subfolders[] = current_folder.listFiles(m_folder_filter);
258 if (subfolders == null) {
259 log.warn("scanFoldersForCurrent: unable to list files from " + current_folder + "; odd characters?");
260 return;
261 }
262
263 for (int i = 0; i < subfolders.length && !stopping(); i++) {
264 File child = subfolders[i];
265
266 scanCurrent(root_folder, child, src_type); // RECURSE!!!
267
268 // If database already contains a reference to the folder,
269 // consider it valid and remove it from the list of folders
270 // to be deleted.
271 referenced.remove( child.getName() );
272 }
273
274 // delete records for all child folders in database not found on disk
275 Iterator it = referenced.values().iterator();
276 while (it.hasNext()) {
277 if (stopping()) return; // detect interruption
278 Folder folder = (Folder)it.next();
279 if (log.isDebugEnabled())
280 log.debug("scanFoldersForCurrent: removing folder reference " + folder);
281 m_fldr_bus.deleteRec(folder.getKey());
282 }
283
284
285 referenced.clear();
286 }
287
288
289 /** scan tunes (or playlists) for the current folder */
290 void scanFilesForCurrent(File current_folder,
291 int folder_id,
292 int src_type) throws IOException, BusException {
293
294 // Obtain a list of files which are already referenced
295 // in the database for the current folder. This list will be culled as
296 // we find the physical files. Any references that
297 // remain after the scan can be considered obsolete zombies which can be
298 // deleted.
299
300 Map referenced = m_mfile_bus.getChildMap(folder_id, src_type, 0, Mfile.NO_LIMIT);
301
302 // obtain list of files (and subfolders) in folder
303 FileFilter filter = (src_type==Source.SRCTYPE_TUNE
304 ? (FileFilter)m_tune_filter
305 : (FileFilter)m_playlist_filter);
306 File children[] = current_folder.listFiles(filter);
307 if (children == null) {
308 log.warn("scanFilesForCurrent: unable to list files from " + current_folder + "; odd characters?");
309 return;
310 }
311
312 // iterate through all children in the folder
313 for (int i = 0; i < children.length && !stopping(); i++) {
314 File child = children[i];
315 String child_name = child.getName();
316
317 if (child.canRead() == false) {
318 log.warn("scanFilesForCurrent: cannot read [" + child + "]; odd filename?");
319 continue;
320 }
321
322 if (child.exists() == false) {
323 log.warn("scanFilesForCurrent: cannot find [" + child + "]; odd filename?");
324 continue;
325 }
326
327 int src_id = scanFile(folder_id, child, referenced);
328 if (src_id > 0) {
329 // if file exists, remove from list of referenced
330 // files to avoid later deletion of source/mfile/tune record
331 if (log.isDebugEnabled())
332 log.debug("scanFilesForCurrent: retaining file: " + child_name);
333 referenced.remove(child_name);
334 }
335 }
336
337 // delete records for all children in database not found on disk
338 Iterator it = referenced.values().iterator();
339 while (it.hasNext()) {
340 if (stopping()) return; // detect interruption
341 Mfile mfile = (Mfile)it.next();
342 int src_id = mfile.getSrcId();
343 if (log.isDebugEnabled())
344 log.debug("scanFilesForCurrent: removing tune or playlist reference " + mfile);
345 m_src_bus.forwardDelete(src_id);
346 }
347
348 referenced.clear();
349 }
350
351 /**
352 * add file if not present in database; otherwise update if changed
353 *
354 * @param folder_id int - Folder in which the file appears.
355 * @param child File - The file on disk.
356 * @param referenced
357 * Map - The list of files (keyed by filename) stored
358 * in the database for this folder.
359 * @return <code>int</code> src_id if file exists; otherwise 0
360 * @exception BusException
361 */
362 int scanFile(int folder_id,
363 File child,
364 Map referenced)
365 throws BusException {
366
367 if (log.isDebugEnabled())
368 log.debug("scanFile: child=" + child
369 + " referenced=" + referenced);
370
371 String child_name = child.getName();
372 Integer src_id = null;
373
374 try {
375 Mfile mfile = (Mfile)referenced.get(child_name);
376 if (mfile != null) {
377 if (log.isDebugEnabled())
378 log.debug("scanFile: child exists in referenced file list");
379
380 src_id = new Integer(mfile.getSrcId());
381
382 // the database already contains a reference to the file,
383 // compare the timestamp to see if we need to update the record
384
385 if (child.lastModified() != mfile.getLastModified()) {
386 if (log.isInfoEnabled())
387 log.info("scanFile: updating newer " + child
388 + " which has a different file date ("
389 + child.lastModified() + " != " + mfile.getLastModified() + ")");
390 m_source_builder.buildAndStore(child);
391 } else {
392 log.debug("scanFile: skipping " + child);
393 }
394 } else {
395 // add as new record in database
396 m_source_builder.buildAndStore(child);
397 }
398 } catch (FileNotFoundException e) {
399 log.warn("file not found for " + child + "; attempting to continue");
400 } catch (BuilderException e) {
401 log.warn("builder-problem with [" + child_name + "]", e);
402 abortIfIntolerant();
403 } catch (IOException e) {
404 log.warn("io-problem with [" + child_name + "]", e);
405 abortIfIntolerant();
406 }
407
408 return src_id != null ? src_id.intValue() : 0;
409 }
410
411
412 /** throw a BusException if the user has configured the scanner as 'Intolerant' of errors. */
413 void abortIfIntolerant() throws BusException {
414 boolean tolerant = false;
415 ScannerSettingCache settings = ScannerSettingCache.getInstance();
416 try {
417 tolerant = settings.getTolerant();
418 } catch (RpcException ignore) {
419 }
420
421 if (tolerant == false)
422 throw new BusException("Fatal scanning error; see scanner settings for more tolerance.");
423 }
424
425 /**
426 * assign a fresh tunecount and duration to each playlist
427 */
428 void refreshPlaylistStats() throws BusException {
429
430 log.debug("refreshPlaylistStats");
431
432 final int BATCH_SIZE = 10;
433
434 //refresh the stats of all playlists except those with offsite data
435 // fix for [ 653246 ] Tree playlist scan updates
436 String filter = "pl_type<>" + Playlist.PLAYLIST_TYPE_STATION;
437
438 Hashtable args = new Hashtable();
439 args.put(Playlist.POPULATE_FILTERABLE, new Boolean(true));
440 args.put(Playlist.POPULATE_FOLDERLIST, new Boolean(true));
441 args.put(Playlist.POPULATE_SOURCELIST, new Boolean(true));
442
443 for (int i = 0; !stopping(); i += BATCH_SIZE) {
444
445 Vector playlists = m_pl_bus.getRecs(filter,
446 null, //unsorted
447 args,
448 i, BATCH_SIZE );
449 if (playlists.size() == 0)
450 break;
451
452 try {
453 m_pl_bus.refreshStats(playlists);
454 } catch (BusException e) {
455 log.error("bus-problem calculating playlist stats", e);
456 }
457 }
458 }
459
460 //
461 // internal data items
462 //
463
464 FolderBus m_fldr_bus ;
465 MfileBus m_mfile_bus;
466 MimeBus m_mime_bus ;
467 PlaylistBus m_pl_bus ;
468 RootBus m_root_bus ;
469 SourceBus m_src_bus ;
470 TuneBus m_tune_bus ;
471
472 SourceBuilder m_source_builder;
473
474 TuneFileFilter m_tune_filter;
475 PlaylistFileFilter m_playlist_filter;
476 FolderFileFilter m_folder_filter;
477
478 /** the present state of the scanner */
479 String state;
480
481 /**
482 * this class is implemented as a singleton
483 */
484 static ScannerDaemon singleton;
485
486 /**
487 * logging object
488 */
489 static Log log = LogFactory.getLog(ScannerDaemon.class);
490 }
491 /*
492 JRECEIVER MODIFIED BSD LICENSE
493
494 Copyright (c) 2001-2002, Reed Esau (reed.esau@pobox.com) All rights reserved.
495
496 Redistribution and use in source and binary forms, with or without
497 modification, are permitted provided that the following conditions are
498 met:
499
500 Redistributions of source code must retain the above copyright notice,
501 this list of conditions and the following disclaimer.
502
503 Redistributions in binary form must reproduce the above copyright notice,
504 this list of conditions and the following disclaimer in the documentation
505 and/or other materials provided with the distribution.
506
507 Neither the name of the JReceiver Project
508 (http://jreceiver.sourceforge.net) nor the names of its contributors may
509 be used to endorse or promote products derived from this software without
510 specific prior written permission.
511
512 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
513 IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
514 THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
515 PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
516 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
517 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
518 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
519 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
520 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
521 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
522 POSSIBILITY OF SUCH DAMAGE.
523 */
524