file deleter fixes

svn path=/trunk/boinc/; revision=756

checkin_notes

@@ -2660,3 +2660,31 @@ David  Dec. 17 2002
         test.inc
     tools/
         backend_lib.C
+
+David   Dec 17 2002
+    - The assimilator takes a -app argument
+    - Added execution of file_delete and assimilator to test_uc.php
+        this should delete all input and output files
+        TODO: check that it actually does
+    - fixed bugs in file_deleter
+
+    todo
+    db/
+        db.h
+        db_mysql.C
+    doc/
+        backend.gif (new)
+        backend.html
+        backend.png (new)
+        work_states.html
+    html_ops/
+        db.inc
+    html_user/
+        user.inc
+    sched/
+        assimilator.C
+        file_deleter.C
+        handle_request.C
+    test/
+        test.inc
+        test_uc.php

db/db.h

@@ -38,16 +38,12 @@ struct PROJECT {
 };
 
 // A compilation target, i.e. a architecture/OS combination.
-// Currently the core client will be given only applications
+// The core client will be given only applications with the same platform
-// that match its platform exactly.
-// This could be generatlized to allow, say, an app compiled
-// for AMD to be run on a "generic Intel x86" core client.
-// In this case we'd need a 1 to N mapping from APP_VERSION to PLATFORM
 //
 struct PLATFORM {
     int id;
     unsigned int create_time;
-    char name[256];             // i.e. "sparc-sun-solaris2.7"
+    char name[256];             // i.e. "sparc-sun-solaris"
 };
 
 // An application.
@@ -341,7 +337,7 @@ extern int db_workunit_lookup_name(WORKUNIT&);
 extern int db_workunit_enum_app_need_validate(WORKUNIT&);
 extern int db_workunit_enum_retry_check_time(WORKUNIT&);
 extern int db_workunit_enum_file_delete_state(WORKUNIT&);
-extern int db_workunit_enum_assimilate_state(WORKUNIT&);
+extern int db_workunit_enum_app_assimilate_state(WORKUNIT&);
 
 extern int db_result_new(RESULT& p);
 extern int db_result(int id, RESULT&);

db/db_mysql.C

@@ -620,12 +620,12 @@ int db_workunit_enum_file_delete_state(WORKUNIT& p) {
     return db_enum(e, &p, TYPE_WORKUNIT, buf);
 }
 
-int db_workunit_enum_assimilate_state(WORKUNIT& p) {
+int db_workunit_enum_app_assimilate_state(WORKUNIT& p) {
     static ENUM e;
     char buf[256];
 
     if (!e.active) {
-        sprintf(buf, "where assimilate_state=%d", p.assimilate_state);
+        sprintf(buf, "where appid=%d and assimilate_state=%d", p.appid, p.assimilate_state);
     }
     return db_enum(e, &p, TYPE_WORKUNIT, buf);
 }

doc/backend.fig

@@ -0,0 +1,164 @@
+#FIG 3.2
+Landscape
+Center
+Inches
+Letter  
+100.00
+Single
+-2
+1200 2
+0 32 #c7b696
+0 33 #effbff
+0 34 #dfcba6
+0 35 #414141
+0 36 #868286
+0 37 #c7c3c7
+0 38 #e7e3e7
+0 39 #8e8e8e
+0 40 #aeaaae
+0 41 #515551
+0 42 #414141
+0 43 #868286
+0 44 #c7c3c7
+0 45 #868286
+0 46 #c7c3c7
+0 47 #e7e3e7
+0 48 #8e8e8e
+0 49 #8e8e8e
+0 50 #414141
+0 51 #868286
+0 52 #c7c3c7
+0 53 #e7e3e7
+0 54 #414141
+0 55 #868286
+0 56 #c7c3c7
+0 57 #e7e3e7
+0 58 #868286
+0 59 #c7c3c7
+0 60 #e7e3e7
+0 61 #c7b696
+0 62 #effbff
+0 63 #dfcba6
+0 64 #c7b696
+0 65 #effbff
+0 66 #dfcba6
+0 67 #aeaaae
+0 68 #515551
+0 69 #8e8e8e
+0 70 #414141
+0 71 #868286
+0 72 #c7c3c7
+0 73 #e7e3e7
+0 74 #414141
+0 75 #868286
+0 76 #c7c3c7
+0 77 #e7e3e7
+0 78 #868286
+0 79 #c7c3c7
+0 80 #e7e3e7
+0 81 #414141
+0 82 #868286
+0 83 #c7c3c7
+0 84 #414141
+0 85 #c7c3c7
+0 86 #e7e3e7
+0 87 #414141
+0 88 #868286
+0 89 #c7c3c7
+0 90 #8e8e8e
+0 91 #414141
+0 92 #868286
+0 93 #c7c3c7
+0 94 #e7e3e7
+0 95 #414141
+0 96 #868286
+0 97 #c7c3c7
+0 98 #e7e3e7
+0 99 #bebebe
+0 100 #515151
+0 101 #000049
+0 102 #797979
+0 103 #303430
+0 104 #414541
+0 105 #414141
+0 106 #868286
+0 107 #c7c3c7
+0 108 #e7e3e7
+6 300 525 2175 1425
+2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
+	 300 975 2175 975 2175 1425 300 1425 300 975
+2 2 0 1 0 30 50 0 20 0.000 0 0 -1 0 0 5
+	 300 525 2175 525 2175 975 300 975 300 525
+4 0 0 50 0 18 14 0.0000 4 210 1575 450 825 work generator\001
+-6
+6 2400 525 3900 1425
+2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
+	 2400 525 3900 525 3900 1425 2400 1425 2400 525
+4 0 0 50 0 18 15 0.0000 4 210 1155 2625 1050 result retry\001
+-6
+6 4125 525 5625 1425
+2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
+	 4125 975 5625 975 5625 1425 4125 1425 4125 975
+2 2 0 1 0 30 50 0 20 0.000 0 0 -1 0 0 5
+	 4125 525 5625 525 5625 975 4125 975 4125 525
+4 0 0 50 0 18 14 0.0000 4 165 900 4350 825 validater\001
+-6
+6 5850 525 7350 1425
+2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
+	 5850 525 7350 525 7350 1425 5850 1425 5850 525
+4 0 0 50 0 18 15 0.0000 4 165 1080 6075 1050 file deleter\001
+-6
+6 7575 525 9150 1425
+2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
+	 7575 975 9150 975 9150 1425 7575 1425 7575 975
+2 2 0 1 0 30 50 0 20 0.000 0 0 -1 0 0 5
+	 7575 525 9150 525 9150 975 7575 975 7575 525
+4 0 0 50 0 18 14 0.0000 4 165 1140 7800 825 assimilator\001
+-6
+6 3000 2925 3975 4200
+5 1 0 1 -1 -1 0 0 -1 0.000 0 1 0 0 3450.000 3375.000 3000 3975 3450 4125 3900 3975
+1 2 0 1 -1 11 0 0 20 0.000 1 0.0000 3471 3144 450 150 3021 3144 3921 3144
+2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 2
+	 3000 3150 3000 3975
+2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 2
+	 3900 3225 3900 3975
+4 1 -1 0 0 18 12 0.0000 6 135 600 3450 3600 BOINC\001
+4 1 -1 0 0 18 12 0.0000 6 135 270 3450 3825 DB\001
+-6
+6 8775 2850 12225 3975
+2 2 0 1 0 30 50 0 20 0.000 0 0 -1 0 0 5
+	 8775 2850 9300 2850 9300 3300 8775 3300 8775 2850
+2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
+	 8775 3450 9300 3450 9300 3900 8775 3900 8775 3450
+4 0 0 50 0 16 14 0.0000 4 210 2775 9450 3300 project-specific components\001
+4 0 0 50 0 16 14 0.0000 4 210 1920 9450 3900 BOINC components\001
+-6
+6 4800 3075 6300 3975
+2 2 0 1 0 7 50 0 20 0.000 0 0 -1 0 0 5
+	 4800 3075 6300 3075 6300 3975 4800 3975 4800 3075
+4 0 0 50 0 18 15 0.0000 4 165 1170 5025 3600 data server\001
+-6
+2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
+	0 0 1.00 60.00 120.00
+	 1725 1425 3075 3000
+2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
+	0 0 1.00 60.00 120.00
+	 2100 1425 4800 3150
+2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
+	0 0 1.00 60.00 120.00
+	 3225 1425 3450 2925
+2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
+	0 0 1.00 60.00 120.00
+	 4875 1425 3675 2925
+2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
+	0 0 1.00 60.00 120.00
+	 6225 1425 3825 2925
+2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
+	0 0 1.00 60.00 120.00
+	 7875 1425 3975 3000
+2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
+	0 0 1.00 60.00 120.00
+	 6675 1425 5700 3000
+2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
+	0 0 1.00 60.00 120.00
+	 8550 1425 6225 3000

doc/backend.html

@@ -8,11 +8,17 @@ for generating work and handling results.
 Some of these components are supplied by BOINC;
 other parts are project- or application-specific:
 
+<br>
+<img vspace=10 src=backend.png>
+<br>
+
 <p>
 <table border=1 cellpadding=8>
+<tr>
 <th>Component</th>
 <th>BOINC-supplied part</th>
 <th>project-supplied part</th>
+</tr>
 <tr>
 <td valign=top>
 <b>Work generator</b>: generates work units, work sequences, results,
@@ -46,6 +52,36 @@ basic logic for validation.</td>
 that compares sets of redundant results.</td>
 </tr>
 <tr>
+<td valign=top><b>Assimilator</b>: parse output files of
+completed results, and take appropriate action
+(record results in a database, and/or generate more work)</td>
+<td valign=top>
+A main program that enumerates unassimilated workunits
+and updates the database.
+</td>
+<td valign=top>
+A function that assimilates a workunit,
+either by processing its canonical result
+or handling an error return.
+</td>
+</tr>
+<tr>
+<td valign=top><b>File deleter</b>: delete input and output files
+when they are no longer needed.</td>
+<td valign=top>A program, <b>file_deleter</b>, that does the job.</td>
+<td valign=top>None.</td>
+</tr>
+</table>
+<p>
+Project that use work sequences will need two additional processes:
+<p>
+<table border=1 cellpadding=8>
+<tr>
+<th>Component</th>
+<th>BOINC-supplied part</th>
+<th>project-supplied part</th>
+</tr>
+<tr>
 <td valign=top><b>Work sequence relocater</b>:
 detects work sequences whose hosts have failed,
 and relocates them to other hosts.</td>
@@ -60,22 +96,4 @@ Similar to result validation, but for work sequences.
 <td valign=top>An application-specific function,
 linked with <b>seq_validate</b>, that compares sets of redundant results.</td>
 </tr>
-<tr>
-<td valign=top><b>Result processing</b>: parse output files of
-completed results, and take appropriate action
-(record results in a database, and/or generate more work)</td>
-<td valign=top>Function for enumerating unprocessed results.
-Functions and programs that handle the details of
-creating workunit, work sequence, and result database records.
-</td>
-<td valign=top>
-The rest.
-</td>
-</tr>
-<tr>
-<td valign=top><b>Garbage collector</b>: delete input and output files
-when they are no longer needed.</td>
-<td valign=top>A program, <b>garbage_collect</b>, that does the job.</td>
-<td valign=top>None.</td>
-</tr>
 </table>

doc/create_project.html

@@ -69,11 +69,12 @@ How to feed work into a BOINC system and collect the results.
 </font>
 <ul>
 <li> <a href=backend.html>The components of a project back end</a>
+<li> <a href=work_states.html>Workunit and result states</a>
 <li> <a href=tools_work.html>Generating work</a>
 <li> <a href=retry.html>Generating result retries</a>
 <li> <a href=validation.html>Result validation and accounting</a>
-<li> <a href=result_process.html>Result processing</a>
+<li> <a href=result_process.html>Assimilation</a>
-<li> <a href=garbage.html>Garbage collection</a>
+<li> <a href=garbage.html>File deletion</a>
 <li> <a href=back_end.html>Back end examples</a>
 </ul>
 

doc/index.html

@@ -8,10 +8,12 @@
 <tr><td width=100%>
 <center>
 <font size=+1>
+<b>
 Berkeley Open Infrastructure for Network Computing
 (BOINC)
 is software that combines many volunteer PCs
 into a parallel supercomputer.
+</b>
 </font>
 </center>
 </td></tr></table>
@@ -74,6 +76,12 @@ width="210" height="62" border="0" alt="SourceForge Logo"></a>
 <br>
 Version 0.06 of BOINC is available on SourceForge.net.
 <br><br>
+<b>December 10, 2002</b>
+<br>
+We have started a beta test of BOINC using
+the Astropulse application.
+Many bugs have been found and fixed.
+<br><br>
 <b>November 25, 2002</b>
 <br>
 Version 0.03 of BOINC is available on SourceForge.net.

doc/intro.html

@@ -32,7 +32,7 @@ Existing applications in common languages (C, C++, Fortran)
 can run as BOINC applications with little or no modification.
 An application can consist of several files
 (e.g. multiple programs and a coordinating script).
-New versions of applications can be deployed with minimal participant involvement.
+New versions of applications can be deployed with no participant involvement.
 <li>
 <b>Security</b>
 <br>
@@ -45,7 +45,14 @@ to protect against the distribution of viruses.
 Projects can have separate scheduling and data servers,
 with multiple servers of each type.
 Clients automatically try alternate servers;
-if all servers are unavailable, they do exponential backoff.
+if all servers are down, clients do exponential backoff
+to avoid flooding the servers when they come back up.
+<li>
+<b>System monitoring tools</b>
+<br>
+BOINC includes a web-based system for displaying time-varying
+measurements (CPU load, network traffic, database table sizes).
+This simplifies the task of diagnosing performance problems.
 <li>
 <b>Open source</b>
 <br>
@@ -80,7 +87,7 @@ Windows, Linux and other Unix systems).
 The client can use multiple CPUs.
 
 <li>
-<b>Web-based administrative interfaces</b>
+<b>Web-based participant interfaces</b>
 <br>
 BOINC provides web-based interfaces for
 account creation, preference editing, and participant status display.
@@ -92,6 +99,6 @@ making it easy to manage large numbers of hosts.
 <br>
 The core client downloads enough work to keep its host
 busy for a user-specifiable amount of time.
-This allows the host to continue to function during project downtimes,
+This can be used to decrease the frequency of connections or to
-and decreases the frequency of connections.
+allow the host to keep working during project downtime.
 </ul>

doc/project.fig

@@ -84,28 +84,6 @@ Single
 0 106 #868286
 0 107 #c7c3c7
 0 108 #e7e3e7
-6 3225 2625 4650 3300
-2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
-	 3225 2625 4650 2625 4650 3300 3225 3300 3225 2625
-4 0 0 50 0 18 14 0.0000 4 210 1155 3375 2925 scheduling\001
-4 0 0 50 0 18 14 0.0000 4 120 660 3600 3150 server\001
--6
-6 4950 3675 5925 4950
-5 1 0 1 -1 -1 0 0 -1 0.000 0 1 0 0 5400.000 4125.000 4950 4725 5400 4875 5850 4725
-1 2 0 1 -1 11 0 0 20 0.000 1 0.0000 5421 3894 450 150 4971 3894 5871 3894
-2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 2
-	 4950 3900 4950 4725
-2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 2
-	 5850 3975 5850 4725
-4 1 -1 0 0 18 12 0.0000 6 135 600 5400 4350 BOINC\001
-4 1 -1 0 0 18 12 0.0000 6 135 270 5400 4575 DB\001
--6
-6 7650 750 8775 1500
-2 2 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 5
-	 7650 750 8775 750 8775 1500 7650 1500 7650 750
-4 0 0 50 0 18 14 0.0000 4 120 780 7800 1350 servers\001
-4 0 0 50 0 18 14 0.0000 4 165 450 7950 1050 data\001
--6
 6 2100 4050 2775 4800
 2 1 0 1 0 3 50 0 20 0.000 0 0 -1 1 0 2
 	0 0 1.00 60.00 120.00
@@ -118,24 +96,34 @@ Single
 	 2400 5550 2400 6075
 4 0 0 50 0 16 14 0.0000 4 165 510 2100 5475 client\001
 -6
-6 4800 2625 6225 3300
+6 3225 4050 4650 4725
 2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
-	 4800 2625 6225 2625 6225 3300 4800 3300 4800 2625
+	 3225 4050 4650 4050 4650 4725 3225 4725 3225 4050
-4 0 0 50 0 18 14 0.0000 4 210 585 5175 2925 utility\001
+4 0 0 50 0 18 14 0.0000 4 210 1155 3375 4350 scheduling\001
-4 0 0 50 0 18 14 0.0000 4 165 1005 5025 3150 programs\001
+4 0 0 50 0 18 14 0.0000 4 120 660 3600 4575 server\001
 -6
-6 6375 2625 7800 3300
+6 6975 3975 8100 4725
+2 2 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 5
+	 6975 3975 8100 3975 8100 4725 6975 4725 6975 3975
+4 0 0 50 0 18 14 0.0000 4 120 780 7125 4575 servers\001
+4 0 0 50 0 18 14 0.0000 4 165 450 7275 4275 data\001
+-6
+6 8475 4050 9900 4725
 2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
-	 6375 2625 7800 2625 7800 3300 6375 3300 6375 2625
+	 8475 4050 9900 4050 9900 4725 8475 4725 8475 4050
-4 0 0 50 0 18 14 0.0000 4 165 465 6900 2925 Web\001
+4 0 0 50 0 18 14 0.0000 4 165 465 9000 4350 Web\001
-4 0 0 50 0 18 14 0.0000 4 165 1020 6600 3150 interfaces\001
+4 0 0 50 0 18 14 0.0000 4 165 1020 8700 4575 interfaces\001
+-6
+6 5175 2175 6150 3450
+5 1 0 1 -1 -1 0 0 -1 0.000 0 1 0 0 5625.000 2625.000 5175 3225 5625 3375 6075 3225
+1 2 0 1 -1 11 0 0 20 0.000 1 0.0000 5646 2394 450 150 5196 2394 6096 2394
+2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 2
+	 5175 2400 5175 3225
+2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 2
+	 6075 2475 6075 3225
+4 1 -1 0 0 18 12 0.0000 6 135 600 5625 2850 BOINC\001
+4 1 -1 0 0 18 12 0.0000 6 135 270 5625 3075 DB\001
 -6
-2 2 0 1 0 30 50 0 20 0.000 0 0 -1 0 0 5
-	 3600 750 6900 750 6900 1500 3600 1500 3600 750
-2 1 0 1 0 3 50 0 20 0.000 0 0 -1 1 1 2
-	0 0 1.00 60.00 120.00
-	0 0 1.00 60.00 120.00
-	 3900 3300 3900 7200
 2 2 0 1 0 30 50 0 20 0.000 0 0 -1 0 0 5
 	 9825 750 10350 750 10350 1200 9825 1200 9825 750
 2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
@@ -154,30 +142,34 @@ Single
 	 3600 7200 8850 7200 8850 8100 3600 8100 3600 7200
 2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
 	0 0 1.00 60.00 120.00
-	 8250 7200 8250 1500
+	 7575 7200 7575 4725
 2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
-	1 1 1.00 60.00 120.00
+	0 0 1.00 60.00 120.00
-	 4350 3300 4950 3750
+	 9900 6225 9225 4725
 2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
-	1 1 1.00 60.00 120.00
+	0 0 1.00 60.00 120.00
-	 5400 3300 5400 3675
+	 3900 7200 3900 4725
 2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
-	1 1 1.00 60.00 120.00
+	0 0 1.00 60.00 120.00
-	 6825 3300 6000 3750
+	 4200 4050 5325 3300
 2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
-	1 1 1.00 60.00 120.00
+	0 0 1.00 60.00 120.00
-	 6309 1488 7125 2625
+	 8850 4050 6075 3225
 2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
-	1 1 1.00 60.00 120.00
+	0 0 1.00 60.00 120.00
-	 5409 1488 5400 2625
+	 5625 1650 5625 2250
+2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
+	 4350 1200 7650 1200 7650 1650 4350 1650 4350 1200
+2 2 0 1 0 30 50 0 20 0.000 0 0 -1 0 0 5
+	 4350 750 7650 750 7650 1200 4350 1200 4350 750
 2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
-	1 1 1.00 60.00 120.00
+	0 0 1.00 60.00 120.00
-	 10081 6123 7500 3300
+	 6825 1650 7500 4050
-4 0 0 50 0 18 14 0.0000 4 210 1740 4350 1275 project back end\001
 4 0 0 50 0 16 14 0.0000 4 210 2775 10500 1200 project-specific components\001
 4 0 0 50 0 16 14 0.0000 4 210 1920 10500 1800 BOINC components\001
 4 0 0 50 0 18 14 0.0000 4 210 1785 4751 6000 application client\001
 4 0 0 50 0 18 14 0.0000 4 165 990 4981 6600 client API\001
 4 0 0 50 0 18 14 0.0000 4 165 1095 5550 7725 core client\001
-4 0 0 50 0 18 14 0.0000 4 210 2355 2100 2400 BOINC server complex\001
 4 0 0 50 0 18 14 0.0000 4 210 1245 9450 6525 participants\001
+4 0 0 50 0 18 14 0.0000 4 210 2355 2325 3675 BOINC server complex\001
+4 0 0 50 0 18 14 0.0000 4 210 1740 2400 900 project back end\001

doc/work_states.html

@@ -0,0 +1,80 @@
+<title>Workunit and result states</title>
+<h2>Workunit and result states</h2>
+
+<p>
+The processing of workunits and results involves
+several independent activities.
+To keep track of these activities,
+workunit and result database records have several "state" fields,
+and their processing can be viewed as the combination
+of several finite-state machines.
+
+<p>
+A workunit has the following state fields:
+<ul>
+<li>
+<b>main_state</b>:
+Initially INIT.
+If a canonical result is found (by validate)
+it transitions to DONE.
+If it's decided that no canonical result can be found
+(by result_retry)
+it transitions to ERROR.
+
+<li>
+<b>file_delete_state</b>:
+Initially INIT.
+When the main state transitions to either DONE or ERROR,
+it transitions to READY,
+indicating that input files can be deleted.
+When file deletion is completed (by file_deleter)
+it transitions to DONE.
+
+<li>
+<b>assimilate_state</b>:
+Initially INIT.
+When the main state transitions to either DONE or ERROR,
+it transitions to READY,
+indicating that the workunit can be assimilated.
+When assimplateion is completed (by assimilator)
+it transitions to DONE.
+
+<li>
+<b>need_validate</b>:
+A boolean, true whenever
+the workunit has a result whose validate state is NEED_CHECK.
+The validate program sets it back to false.
+
+</ul>
+
+
+<p>
+
+A result has the following state fields:
+<ul>
+<li>
+<b>server_state</b>:
+Initially UNSENT.
+Becomes IN_PROGRESS when the result has been sent to a client.
+Becomes DONE or ERROR if the client returns the result,
+or TIMEOUT if result_retry gives up on the result.
+<li>
+<b>client_state</b>:
+Records the client state (upload, process, or download)
+where an error occurred.
+Not relevant here.
+<li>
+<b>file_delete_state</b>:
+Initially INIT.
+When the parent workunit becomes ERROR, becomes READY,
+indication that the output files can be deleted.
+When the parent workunit becomes DONE, becomes READY
+except for the canonical result, which remains INIT.
+When the output files have been deleted (by file_deleter) becomes DONE.
+<li>
+<b>validate_state</b>:
+Initially INITIAL.
+When the result is returned, becomes NEED_CHECK.
+When a canonical result has been found for the workunit,
+becomes either VALID or INVALID.
+</ul>

html/ops/db.inc

@@ -37,8 +37,8 @@ function show_platform($platform) {
     start_table();
     row("ID", $platform->id);
     row("Created", time_str($platform->create_time));
-    row("Platform Name", $platform->name);
+    row("Name", $platform->name);
-    row("","<a href=db.php?show=app_version&plat_id=$platform->id>Application Versions for Platform $platform->name </a>");
+    row("","<a href=db.php?show=app_version&plat_id=$platform->id>App versions for this platform</a>");
     end_table();
 }
 
@@ -46,9 +46,9 @@ function show_app($app) {
     start_table();
     row("ID", $app->id);
     row("Created", time_str($app->create_time));
-    row("Application Name", $app->name);
+    row("Name", $app->name);
-    row("","<a href=db.php?show=app_version&app_id=$app->id>Application Versions for Application $app->name</a>");
+    row("","<a href=db.php?show=app_version&app_id=$app->id>App Versions for this application</a>");
-    row("","<a href=db.php?show=workunit&app_id=$app->id>Workunits for this Application</a>");
+    row("","<a href=db.php?show=workunit&app_id=$app->id>Workunits for this application</a>");
     end_table();
 }
 
@@ -109,9 +109,17 @@ function show_host($host) {
 
 }
 
-function retry_state_str($s) {
+function main_state_str($s) {
     switch($s) {
-    case 0: return "Default";
+    case 0: return "Initial";
+    case 1: return "Done";
+    case 2: return "Error";
+    }
+}
+
+function error_str($s) {
+    switch($s) {
+    case 0: return "No error";
     case 1: return "Send failed";
     case 2: return "Too many errors (may have bug)";
     case 3: return "Too many done (may be nondeterministic)";
@@ -119,6 +127,22 @@ function retry_state_str($s) {
     return "unknown";
 }
 
+function assimilate_state_str($s) {
+    switch($s) {
+    case 0: return "Initial";
+    case 1: return "Ready to assimilate";
+    case 2: return "Assimilated";
+    }
+}
+
+function file_delete_state_str($s) {
+    switch($s) {
+    case 0: return "Initial";
+    case 1: return "Ready to delete";
+    case 2: return "Deleted";
+    }
+}
+
 function show_workunit($wu,$show_xml_doc) {
     start_table();
     row("Created", time_str($wu->create_time));
@@ -136,8 +160,11 @@ function show_workunit($wu,$show_xml_doc) {
     row("Canonical resultid", $wu->canonical_resultid);
     row("Canonical credit", $wu->canonical_credit);
     row("Retry check time", time_str($wu->retry_check_time));
-    row("Retry state", retry_state_str($wu->state));
     row("Delay bound", $wu->delay_bound);
+    row("Main state", main_state_str($wu->main_state));
+    row("Error", error_str($wu->error));
+    row("File delete state", file_delete_state_str($wu->file_delete_state));
+    row("Assimilation state", assimilate_state_str($wu->assimilate_state));
     row("","<a href=db.php?show=result&wu_id=$wu->id>Show associated results</a>");
     end_table();
     echo "<p>";
@@ -168,16 +195,16 @@ function validate_state_str($s) {
 function show_result($result,$show_xml_docs,$show_stderr,$show_times) {
     start_table();
     if ($show_times) {
-        row("created", time_str($result->create_time));
+        row("Created", time_str($result->create_time));
-        row("sent", time_str($result->sent_time));
+        row("Sent", time_str($result->sent_time));
-        row("received", time_str($result->received_time));
+        row("Received", time_str($result->received_time));
     }
     row("Name", $result->name);
     row("Workunit", "<a href=db.php?show=workunit&id=$result->workunitid>" . wu_name_by_id($result->workunitid) . "</a>" );
-    row("state", res_state_string($result->server_state));
+    row("Server state", res_state_string($result->server_state));
     row("Deadline", time_str($result->report_deadline));
     row("Host ID", "<a href=db.php?show=host&id=$result->hostid>" . host_name_by_id($result->hostid) . "</a>");
-    row("Client_state: ", $result->client_state);
+    row("Client state", $result->client_state);
     row("CPU time", $result->cpu_time);
     if ($show_xml_docs) {
         row("XML doc in", "<pre>".htmlspecialchars($result->xml_doc_in)."</pre>");
@@ -186,42 +213,42 @@ function show_result($result,$show_xml_docs,$show_stderr,$show_times) {
     if ($show_stderr) {
         row("stderr out", "<pre>".htmlspecialchars($result->stderr_out)."</pre>");
     }
-    row("batch", $result->batch);
+    row("Batch", $result->batch);
-    row("project state", $result->project_state);
+    row("File delete state", file_delete_state_str($result->file_delete_state));
     row("Validate state", validate_state_str($result->validate_state));
     row("claimed credit", $result->claimed_credit);
-    row("granted credit", $result->granted_credit);
+    row("Granted credit", $result->granted_credit);
     end_table();
     echo "<p>";
 }
 
 function show_user($user) {
     start_table();
-    row("created", time_str($user->create_time));
+    row("Created", time_str($user->create_time));
-    row("total credit", $user->total_credit);
+    row("Total credit", $user->total_credit);
-    row("recent averaged credit:", $user->expavg_credit);
+    row("Recent averaged credit:", $user->expavg_credit);
-    row("name", $user->name);
+    row("Name", $user->name);
-    row("email address", $user->email_addr);
+    row("Email address", $user->email_addr);
-    row("country", $user->country);
+    row("Country", $user->country);
-    row("postal code", $user->postal_code);
+    row("Postal code", $user->postal_code);
-    row("total credit", $user->total_credit);
+    row("Total credit", $user->total_credit);
-    row("average credit", $user->expavg_credit);
+    row("Average credit", $user->expavg_credit);
-    row("last average time", time_str($user->expavg_time));
+    row("Last average time", time_str($user->expavg_time));
     end_table();
 }
 
 function team_type_string($s) {
     switch ($s) {
-        case 1: return "Small Company";
+    case 1: return "Small Company";
-        case 2: return "Medium Company";
+    case 2: return "Medium Company";
-        case 3: return "Large Company";
+    case 3: return "Large Company";
-        case 4: return "Club";
+    case 4: return "Club";
-        case 5: return "Primary School";
+    case 5: return "Primary School";
-        case 6: return "Secondary School";
+    case 6: return "Secondary School";
-        case 7: return "Junior College";
+    case 7: return "Junior College";
-        case 8: return "University or Department";
+    case 8: return "University or Department";
-        case 9: return "Government Agency";
+    case 9: return "Government Agency";
-        default: return "Unknown";
+    default: return "Unknown";
     }
 }
 

html/user/user.inc

@@ -1,10 +1,11 @@
 <?php
 
+// show a user summary.  Contains only public information
+//
 function show_user_stats($user) {
     echo TABLE2."\n";
-    echo "<tr>".TD2.LG_FONT."<b>User Stats:</b></font></td></tr>\n";
+    row("<b>Name:   </b>", $user->name);
-    row("<b>User Name:   </b>", $user->name);
+    $row = sprintf("<b>%s Participant since:   </b>", $project);
-    $row = sprintf("<b>%s user since:   </b>", $project);
     row($row, time_str($user->create_time));
     row("<b>Total credit:   </b>", $user->total_credit);
     row("<b>Recent average credit:   </b>", $user->expavg_credit);
@@ -18,6 +19,8 @@ function show_user_stats($user) {
     echo "</table>\n";
 }
 
+// show private user info
+//
 function show_user_profile($user) {
     echo TABLE2."\n";
     echo "<tr>".TD2.LG_FONT."<b>User Information:</b></font></td></tr>\n";
@@ -48,10 +51,10 @@ function show_hosts($user) {
 
 function show_host_brief($host) {
     echo "<tr>
-              <td><a href=show_host_detail.php?hostid=$host->id>$host->domain_name</a></td>
+        <td><a href=show_host_detail.php?hostid=$host->id>$host->domain_name</a></td>
-              <td>$host->total_credit</td>
+        <td>$host->total_credit</td>
-              <td>$host->expavg_credit</td>
+        <td>$host->expavg_credit</td>
-          </tr>";
+        </tr>";
 }
 
 // show a summary of the user.

sched/assimilator.C

@@ -10,14 +10,15 @@ CONFIG config;
 
 // return nonzero if did anything
 //
-bool do_pass() {
+bool do_pass(APP app) {
     WORKUNIT wu;
     RESULT result;
     bool did_something = false;
     int retval;
 
+    wu.appid = app.id;
     wu.assimilate_state = ASSIMILATE_READY;
-    while (db_workunit_enum_assimilate_state(wu)) {
+    while (!db_workunit_enum_app_assimilate_state(wu)) {
         did_something = true;
         switch(wu.main_state) {
         case WU_MAIN_STATE_INIT:
@@ -50,6 +51,7 @@ bool do_pass() {
 int main(int argc, char** argv) {
     int retval;
     bool asynch = false, one_pass = false;
+    APP app;
     int i;
 
     for (i=1; i<argc; i++) {
@@ -57,6 +59,8 @@ int main(int argc, char** argv) {
             asynch = true;
         } else if (!strcmp(argv[i], "-one_pass")) {
             one_pass = true;
+        } else if (!strcmp(argv[i], "-app")) {
+            strcpy(app.name, argv[++i]);
         } else {
             fprintf(stderr, "Unrecognized arg: %s\n", argv[i]);
         }
@@ -75,11 +79,20 @@ int main(int argc, char** argv) {
     }
 
     retval = db_open(config.db_name, config.db_passwd);
+    if (retval) {
+        fprintf(stderr, "Can't open DB\n");
+        exit(1);
+    }
+    retval = db_app_lookup_name(app);
+    if (retval) {
+        fprintf(stderr, "Can't find app\n");
+        exit(1);
+    }
     if (one_pass) {
-        do_pass();
+        do_pass(app);
     } else {
         while (1) {
-            if (!do_pass()) sleep(10);
+            if (!do_pass(app)) sleep(10);
         }
     }
 }

sched/file_deleter.C

@@ -16,9 +16,7 @@ int wu_delete_files(WORKUNIT& wu) {
     p = strtok(wu.xml_doc, "\n");
     strcpy(filename, "");
     while (p) {
-        p = strtok(0, "\n");
         if (parse_str(p, "<name>", filename, sizeof(filename))) {
-            continue;
         } else if (match_tag(p, "<file_info>")) {
             no_delete = false;
             strcpy(filename, "");
@@ -27,9 +25,11 @@ int wu_delete_files(WORKUNIT& wu) {
         } else if (match_tag(p, "</file_info>")) {
             if (!no_delete) {
                 sprintf(pathname, "%s/%s", config.download_dir, filename);
+                printf("deleting %s\n", pathname);
                 unlink(pathname);
             }
         }
+        p = strtok(0, "\n");
     }
     return 0;
 }
@@ -41,9 +41,7 @@ int result_delete_files(RESULT& result) {
 
     p = strtok(result.xml_doc_in, "\n");
     while (p) {
-        p = strtok(0, "\n");
         if (parse_str(p, "<name>", filename, sizeof(filename))) {
-            continue;
         } else if (match_tag(p, "<file_info>")) {
             no_delete = false;
             strcpy(filename, "");
@@ -52,9 +50,11 @@ int result_delete_files(RESULT& result) {
         } else if (match_tag(p, "</file_info>")) {
             if (!no_delete) {
                 sprintf(pathname, "%s/%s", config.upload_dir, filename);
+                printf("deleting %s\n", pathname);
                 unlink(pathname);
             }
         }
+        p = strtok(0, "\n");
     }
     return 0;
 }
@@ -67,7 +67,7 @@ bool do_pass() {
     bool did_something = false;
 
     wu.file_delete_state = FILE_DELETE_READY;
-    while (db_workunit_enum_file_delete_state(wu)) {
+    while (!db_workunit_enum_file_delete_state(wu)) {
         did_something = true;
         wu_delete_files(wu);
         wu.file_delete_state = FILE_DELETE_DONE;
@@ -75,7 +75,7 @@ bool do_pass() {
     }
 
     result.file_delete_state = FILE_DELETE_READY;
-    while (db_result_enum_file_delete_state(result)) {
+    while (!db_result_enum_file_delete_state(result)) {
         did_something = true;
         result_delete_files(result);
         result.file_delete_state = FILE_DELETE_DONE;

sched/handle_request.C

@@ -363,9 +363,6 @@ int handle_results(
                 if (retval) {
                     fprintf(stderr, "Can't update WU\n");
                 }
-                fprintf(stderr,
-                    "BOINC scheduler: set need_validate for WU %d\n", wu.id
-                );
             }
         }
     }

test/test.inc

@@ -248,6 +248,8 @@ class Project {
         PassThru("cp $source_dir/sched/feeder $this->project_dir/cgi/");
         PassThru("cp $source_dir/sched/result_retry $this->project_dir/cgi/");
         PassThru("cp $source_dir/sched/validate_test $this->project_dir/cgi/");
+        PassThru("cp $source_dir/sched/file_deleter $this->project_dir/cgi/");
+        PassThru("cp $source_dir/sched/assimilator $this->project_dir/cgi/");
         PassThru("cp $source_dir/html_ops/stripchart.cgi $this->project_dir/cgi/");
         $f = fopen("$this->project_dir/cgi/config.xml", "w");
         fputs($f, "<config>\n");
@@ -420,6 +422,18 @@ class Project {
         PassThru("cd $this->project_dir/cgi; ./validate_test -one_pass -app $app->name -quorum $quorum > validate_out");
     }
 
+    // do one pass of file_deleter
+    //
+    function file_delete() {
+        PassThru("cd $this->project_dir/cgi; ./file_deleter -one_pass > file_deleter_out");
+    }
+
+    // do one pass of assimilator
+    //
+    function assimilate($app) {
+        PassThru("cd $this->project_dir/cgi; ./assimilator -one_pass -app $app->name > assimilator_out");
+    }
+
     // this should stop the feeder and any other daemons
     //
     function stop() {

test/test_uc.php

@@ -49,4 +49,9 @@
     $project->check_results(2, $result);
     $project->compare_file("uc_wu_0_0", "uc_correct_output");
     $project->compare_file("uc_wu_1_0", "uc_correct_output");
+
+    $project->file_delete();
+    // input file should be gone here
+    $project->assimilate($app);
+    $project->file_delete();
 ?>

todo

@@ -1,36 +1,130 @@
-write garbage collector
+-----------------------
-
+HIGH-PRIORITY (should do for beta test)
-prevent 2 core clients from running at once
+-----------------------
-
-support for HTTP and SOCKS proxies
 make get_local_ip_addr() work in all cases
 est_time_to_completion doesn't work for non-running tasks
-get timezone working on all platforms
-get benchmark working as separate process
-    must do this before send first scheduler request
 
-validate in crontab
+run backend programs (validate/file_deleter/assimilate)
+    from crontab; document
 
 Windows client
     make mini-logo
     fix "unable to calculate"
     completed results: show final CPU time
+
+Messages from core client
+    decide what messages should be shown to user, and how
+    log file?  GUI?  dialog?
+
+-----------------------
+MEDIUM-PRIORITY (should do before public release)
 -----------------------
 
-use https for login (don't sent account ID or password in clear)
-
 protect project admin web pages (htaccess)
+get timezone working on all platforms
 
 Deadline mechanism for results
     - use in result dispatching
     - use in file uploading (decide what to upload next)
     - use in deciding when to make scheduler RPC (done already?)
 
-create "result adder" program that detects WUs that don't have
+Testing framework
-    a canonical result yet, and should,
+    better mechanisms to model server/client/communication failure
-    and creates more results for them.
+    better mechanisms to simulate large load
-    This should detect situations where we're getting lots
+    do client/server on separate hosts?
-    of error results, and skip over those WUs
+
+Delete files if needed to honor disk usage constraint
+    inform user if this happens
+
+implement max bytes/sec network preferences
+
+Global preferences
+    implement disk usage prefs
+    time-of-day prefs?
+    test propagation mechanism
+        set up multi-project, multi-host test;
+        change global prefs at one web site,
+        make sure they propagate to all hosts
+    limit on frequency of disk writes?
+    max net traffic per day?
+        implement in client
+
+Per-project preferences
+    test project-specific prefs
+        make example web edit pages
+        make app that uses them
+    set up a test with multiple projects
+        test "add project" feature, GUI and cmdline
+        test resource share mechanism
+
+CPU benchmarking
+    review CPU benchmarks - do they do what we want?
+    what to do when tests show hardware problem?
+    How should we weight factors for credit?
+    run CPU tests unobtrusively, periodically
+    check that on/conn/active fracs are maintainted correctly
+    check that bandwidth is measured correctly
+    measure disk/mem size on all platforms
+    get timezone to work
+
+CPU accounting in the presence of checkpoint/restart
+    test
+
+Test nslots > 1
+
+Redundancy checking and validation
+    test the validation mechanism
+    make sure credit is granted correctly
+    make sure average, total credit maintained correctly for user, host
+
+Windows screensaver functionality
+    idle-only behavior without screensaver - test
+
+Data transfer
+    make sure restart of downloads works
+    make sure restart of uploads works
+    test download/upload with multiple data servers
+        make sure it tries servers in succession,
+        does exponential backoff if all fail
+    review and document prioritization of transfers
+    review protocol; make sure error returns are possible and handled correctly
+
+Scheduler
+    Should dispatch results based on deadline?
+    test that scheduler estimates WU completion time correctly
+    test that scheduler sends right amount of work
+    test that client estimates remaining work correctly,
+        requests correct # of seconds
+    test that hi/low water mark system works
+    test that scheduler sends only feasible WUs
+
+Scheduler RPC
+    formalize notion of "permanent failure" (e.g. can't download file)
+    report perm failures to scheduler, record in DB
+    make sure RPC backoff is done for any perm failure
+        (in general, should never make back-to-back RPCs to a project)
+    make sure that client eventually reloads master URL
+
+Application graphics
+    finish design, implementation, doc, testing
+        size, frame rate, whether to generate
+    
+Work generation
+    generation of upload signature is very slow
+
+prevent file_xfer->req1 from overflowing. This problems seems to be
+    happening when the file_upload_handler returnes a message to the
+    client that is large. This causes project->parsefile to get wrong
+    input and so on.
+
+test HTTP redirect mechanism for all types of ops
+
+Add batch features to ops web
+-----------------------
+LONG-TERM IDEAS AND PROJECTS
+-----------------------
+
+use https for login (don't sent account ID or password in clear)
 
 CPU benchmarking
     This should be done by a pseudo-application
@@ -40,14 +134,6 @@ CPU benchmarking
     benchmarking programs (e.g. for graphics coprocessor)
     or project-specific programs.
 
-Testing framework
-    better mechanisms to model server/client/communication failure
-    better mechanisms to simulate large load
-    do client/server on separate hosts?
-
-Add "garbage-collect state" fields to WU, result
-    whether deletion of files is OK, is done
-
 investigate binary diff mechanism for updating persistent files
 
 verify support for > 4 GB files everywhere
@@ -55,11 +141,6 @@ verify support for > 4 GB files everywhere
 use FTP instead of HTTP for file xfer??
     measure speed diff
 
-Delete files if needed to honor disk usage constraint
-    inform user if this happens
-
-implement max bytes/sec network preferences
-
 Local scheduling
     more intelligent decision about when/what to work on
     - monitor VM situation, run small-footprint programs
@@ -85,99 +166,12 @@ Include account ID in URL for file xfers
     This would let you verify network xfers by scanning web logs
     (could use that to give credit for xfers)
 
-Global preferences
-    implement disk usage prefs
-    time-of-day prefs?
-    test propagation mechanism
-        set up multi-project, multi-host test;
-        change global prefs at one web site,
-        make sure they propagate to all hosts
-    limit on frequency of disk writes?
-    max net traffic per day?
-        implement in client
-
-Per-project preferences
-    test project-specific prefs
-        make example web edit pages
-        make app that uses them
-    set up a test with multiple projects
-        test "add project" feature, GUI and cmdline
-        test resource share mechanism
-
-Proxies
-    work through HTTP, Socks proxies
-    look at other open-source code (Mozilla?)
-
-Documentation
-    simplify/finish docs on server installation
-    get rid of text in INSTALL and INSTALL_CLIENT;
-        they should just refer to .html files
-
-Testing in general
-    figure out how to set up multi-project, multi-host tests
-        from a single script
-    automate some simple test cases
-
-Messages from core client
-    decide what messages should be shown to user, and how
-    log file?  GUI?  dialog?
-
-CPU benchmarking
-    review CPU benchmarks - do they do what we want?
-    what to do when tests show hardware problem?
-    How should we weight factors for credit?
-    run CPU tests unobtrusively, periodically
-    check that on/conn/active fracs are maintainted correctly
-    check that bandwidth is measured correctly
-    measure disk/mem size on all platforms
-    get timezone to work
-
 WU/result sequence mechanism
     design/implement/document
 
 Multiple application files
     document, test
 
-CPU accounting in the presence of checkpoint/restart
-    test
-
-Test nslots > 1
-
-Redundancy checking and validation
-    test the validation mechanism
-    make sure credit is granted correctly
-    make sure average, total credit maintained correctly for user, host
-
-Scheduler RPC
-    formalize notion of "permanent failure" (e.g. can't download file)
-    report perm failures to scheduler, record in DB
-    make sure RPC backoff is done for any perm failure
-        (in general, should never make back-to-back RPCs to a project)
-    make sure that client eventually reloads master URL
-
-Data transfer
-    make sure restart of downloads works
-    make sure restart of uploads works
-    test download/upload with multiple data servers
-        make sure it tries servers in succession,
-        does exponential backoff if all fail
-    review and document prioritization of transfers
-    review protocol; make sure error returns are possible and handled correctly
-
-Application graphics
-    finish design, implementation, doc, testing
-        size, frame rate, whether to generate
-    
-Work generation
-    generation of upload signature is very slow
-
-Windows GUI
-    finish design/implement
-    display credit info, team
-
-Windows screensaver functionality
-    idle-only behavior without screensaver - test
-
 Versioning
     think through issues involved in:
     compatibility of core client and scheduling server
@@ -187,29 +181,10 @@ Versioning
     Need version numbers for protocols/interfaces?
     What messages to show user?  Project?
 
-Scheduler
-    Should dispatch results based on deadline?
-    test that scheduler estimates WU completion time correctly
-    test that scheduler sends right amount of work
-    test that client estimates remaining work correctly,
-        requests correct # of seconds
-    test that hi/low water mark system works
-    test that scheduler sends only feasible WUs
-
 Persistent files
     test
     design/implement test reporting, retrieval mechanisms
     (do this using WU/results with null application?)
 
-User HTML
-    leader boards
-
 NET_XFER_SET
     review logic; prevent one stream for starving others
-
-test HTTP redirect mechanism for all types of ops
-
-prevent file_xfer->req1 from overflowing. This problems seems to be
-happening when the file_upload_handler returnes a message to the
-client that is large. This causes project->parsefile to get wrong
-input and so on.