Rooba
/
boinc
mirror of https://github.com/BOINC/boinc.git
1
0
Fork 0
Code Issues 1 Packages Projects Releases Wiki Activity

file deleter fixes 

svn path=/trunk/boinc/; revision=756
This commit is contained in:
David Anderson 2002-12-18 01:34:51 +00:00
parent 452f91c6c3
commit 8567738db7
20 changed files with 604 additions and 276 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
checkin_notes
View File

@ -2660,3 +2660,31 @@ David Dec. 17 2002
test.inc
tools/
backend_lib.C
David Dec 17 2002
- The assimilator takes a -app argument
- Added execution of file_delete and assimilator to test_uc.php
this should delete all input and output files
TODO: check that it actually does
- fixed bugs in file_deleter
todo
db/
db.h
db_mysql.C
doc/
backend.gif (new)
backend.html
backend.png (new)
work_states.html
html_ops/
db.inc
html_user/
user.inc
sched/
assimilator.C
file_deleter.C
handle_request.C
test/
test.inc
test_uc.php

10
db/db.h
View File

@ -38,16 +38,12 @@ struct PROJECT {
};
// A compilation target, i.e. a architecture/OS combination.
// Currently the core client will be given only applications
// that match its platform exactly.
// This could be generatlized to allow, say, an app compiled
// for AMD to be run on a "generic Intel x86" core client.
// In this case we'd need a 1 to N mapping from APP_VERSION to PLATFORM
// The core client will be given only applications with the same platform
//
struct PLATFORM {
int id;
unsigned int create_time;
char name[256]; // i.e. "sparc-sun-solaris2.7"
char name[256]; // i.e. "sparc-sun-solaris"
};
// An application.
@ -341,7 +337,7 @@ extern int db_workunit_lookup_name(WORKUNIT&);
extern int db_workunit_enum_app_need_validate(WORKUNIT&);
extern int db_workunit_enum_retry_check_time(WORKUNIT&);
extern int db_workunit_enum_file_delete_state(WORKUNIT&);
extern int db_workunit_enum_assimilate_state(WORKUNIT&);
extern int db_workunit_enum_app_assimilate_state(WORKUNIT&);
extern int db_result_new(RESULT& p);
extern int db_result(int id, RESULT&);

4
db/db_mysql.C
View File

@ -620,12 +620,12 @@ int db_workunit_enum_file_delete_state(WORKUNIT& p) {
return db_enum(e, &p, TYPE_WORKUNIT, buf);
}
int db_workunit_enum_assimilate_state(WORKUNIT& p) {
int db_workunit_enum_app_assimilate_state(WORKUNIT& p) {
static ENUM e;
char buf[256];
if (!e.active) {
sprintf(buf, "where assimilate_state=%d", p.assimilate_state);
sprintf(buf, "where appid=%d and assimilate_state=%d", p.appid, p.assimilate_state);
}
return db_enum(e, &p, TYPE_WORKUNIT, buf);
}

164
doc/backend.fig Normal file
View File

@ -0,0 +1,164 @@
#FIG 3.2
Landscape
Center
Inches
Letter
100.00
Single
-2
1200 2
0 32 #c7b696
0 33 #effbff
0 34 #dfcba6
0 35 #414141
0 36 #868286
0 37 #c7c3c7
0 38 #e7e3e7
0 39 #8e8e8e
0 40 #aeaaae
0 41 #515551
0 42 #414141
0 43 #868286
0 44 #c7c3c7
0 45 #868286
0 46 #c7c3c7
0 47 #e7e3e7
0 48 #8e8e8e
0 49 #8e8e8e
0 50 #414141
0 51 #868286
0 52 #c7c3c7
0 53 #e7e3e7
0 54 #414141
0 55 #868286
0 56 #c7c3c7
0 57 #e7e3e7
0 58 #868286
0 59 #c7c3c7
0 60 #e7e3e7
0 61 #c7b696
0 62 #effbff
0 63 #dfcba6
0 64 #c7b696
0 65 #effbff
0 66 #dfcba6
0 67 #aeaaae
0 68 #515551
0 69 #8e8e8e
0 70 #414141
0 71 #868286
0 72 #c7c3c7
0 73 #e7e3e7
0 74 #414141
0 75 #868286
0 76 #c7c3c7
0 77 #e7e3e7
0 78 #868286
0 79 #c7c3c7
0 80 #e7e3e7
0 81 #414141
0 82 #868286
0 83 #c7c3c7
0 84 #414141
0 85 #c7c3c7
0 86 #e7e3e7
0 87 #414141
0 88 #868286
0 89 #c7c3c7
0 90 #8e8e8e
0 91 #414141
0 92 #868286
0 93 #c7c3c7
0 94 #e7e3e7
0 95 #414141
0 96 #868286
0 97 #c7c3c7
0 98 #e7e3e7
0 99 #bebebe
0 100 #515151
0 101 #000049
0 102 #797979
0 103 #303430
0 104 #414541
0 105 #414141
0 106 #868286
0 107 #c7c3c7
0 108 #e7e3e7
6 300 525 2175 1425
2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
300 975 2175 975 2175 1425 300 1425 300 975
2 2 0 1 0 30 50 0 20 0.000 0 0 -1 0 0 5
300 525 2175 525 2175 975 300 975 300 525
4 0 0 50 0 18 14 0.0000 4 210 1575 450 825 work generator\001
-6
6 2400 525 3900 1425
2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
2400 525 3900 525 3900 1425 2400 1425 2400 525
4 0 0 50 0 18 15 0.0000 4 210 1155 2625 1050 result retry\001
-6
6 4125 525 5625 1425
2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
4125 975 5625 975 5625 1425 4125 1425 4125 975
2 2 0 1 0 30 50 0 20 0.000 0 0 -1 0 0 5
4125 525 5625 525 5625 975 4125 975 4125 525
4 0 0 50 0 18 14 0.0000 4 165 900 4350 825 validater\001
-6
6 5850 525 7350 1425
2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
5850 525 7350 525 7350 1425 5850 1425 5850 525
4 0 0 50 0 18 15 0.0000 4 165 1080 6075 1050 file deleter\001
-6
6 7575 525 9150 1425
2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
7575 975 9150 975 9150 1425 7575 1425 7575 975
2 2 0 1 0 30 50 0 20 0.000 0 0 -1 0 0 5
7575 525 9150 525 9150 975 7575 975 7575 525
4 0 0 50 0 18 14 0.0000 4 165 1140 7800 825 assimilator\001
-6
6 3000 2925 3975 4200
5 1 0 1 -1 -1 0 0 -1 0.000 0 1 0 0 3450.000 3375.000 3000 3975 3450 4125 3900 3975
1 2 0 1 -1 11 0 0 20 0.000 1 0.0000 3471 3144 450 150 3021 3144 3921 3144
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 2
3000 3150 3000 3975
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 2
3900 3225 3900 3975
4 1 -1 0 0 18 12 0.0000 6 135 600 3450 3600 BOINC\001
4 1 -1 0 0 18 12 0.0000 6 135 270 3450 3825 DB\001
-6
6 8775 2850 12225 3975
2 2 0 1 0 30 50 0 20 0.000 0 0 -1 0 0 5
8775 2850 9300 2850 9300 3300 8775 3300 8775 2850
2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
8775 3450 9300 3450 9300 3900 8775 3900 8775 3450
4 0 0 50 0 16 14 0.0000 4 210 2775 9450 3300 project-specific components\001
4 0 0 50 0 16 14 0.0000 4 210 1920 9450 3900 BOINC components\001
-6
6 4800 3075 6300 3975
2 2 0 1 0 7 50 0 20 0.000 0 0 -1 0 0 5
4800 3075 6300 3075 6300 3975 4800 3975 4800 3075
4 0 0 50 0 18 15 0.0000 4 165 1170 5025 3600 data server\001
-6
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
0 0 1.00 60.00 120.00
1725 1425 3075 3000
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
0 0 1.00 60.00 120.00
2100 1425 4800 3150
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
0 0 1.00 60.00 120.00
3225 1425 3450 2925
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
0 0 1.00 60.00 120.00
4875 1425 3675 2925
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
0 0 1.00 60.00 120.00
6225 1425 3825 2925
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
0 0 1.00 60.00 120.00
7875 1425 3975 3000
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
0 0 1.00 60.00 120.00
6675 1425 5700 3000
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
0 0 1.00 60.00 120.00
8550 1425 6225 3000

54
doc/backend.html
View File

@ -8,11 +8,17 @@ for generating work and handling results.
Some of these components are supplied by BOINC;
other parts are project- or application-specific:
<br>
<img vspace=10 src=backend.png>
<br>
<p>
<table border=1 cellpadding=8>
<tr>
<th>Component</th>
<th>BOINC-supplied part</th>
<th>project-supplied part</th>
</tr>
<tr>
<td valign=top>
<b>Work generator</b>: generates work units, work sequences, results,
@ -46,6 +52,36 @@ basic logic for validation.</td>
that compares sets of redundant results.</td>
</tr>
<tr>
<td valign=top><b>Assimilator</b>: parse output files of
completed results, and take appropriate action
(record results in a database, and/or generate more work)</td>
<td valign=top>
A main program that enumerates unassimilated workunits
and updates the database.
</td>
<td valign=top>
A function that assimilates a workunit,
either by processing its canonical result
or handling an error return.
</td>
</tr>
<tr>
<td valign=top><b>File deleter</b>: delete input and output files
when they are no longer needed.</td>
<td valign=top>A program, <b>file_deleter</b>, that does the job.</td>
<td valign=top>None.</td>
</tr>
</table>
<p>
Project that use work sequences will need two additional processes:
<p>
<table border=1 cellpadding=8>
<tr>
<th>Component</th>
<th>BOINC-supplied part</th>
<th>project-supplied part</th>
</tr>
<tr>
<td valign=top><b>Work sequence relocater</b>:
detects work sequences whose hosts have failed,
and relocates them to other hosts.</td>
@ -60,22 +96,4 @@ Similar to result validation, but for work sequences.
<td valign=top>An application-specific function,
linked with <b>seq_validate</b>, that compares sets of redundant results.</td>
</tr>
<tr>
<td valign=top><b>Result processing</b>: parse output files of
completed results, and take appropriate action
(record results in a database, and/or generate more work)</td>
<td valign=top>Function for enumerating unprocessed results.
Functions and programs that handle the details of
creating workunit, work sequence, and result database records.
</td>
<td valign=top>
The rest.
</td>
</tr>
<tr>
<td valign=top><b>Garbage collector</b>: delete input and output files
when they are no longer needed.</td>
<td valign=top>A program, <b>garbage_collect</b>, that does the job.</td>
<td valign=top>None.</td>
</tr>
</table>

BIN
doc/backend.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 5.9 KiB

5
doc/create_project.html
View File

@ -69,11 +69,12 @@ How to feed work into a BOINC system and collect the results.
</font>
<ul>
<li> <a href=backend.html>The components of a project back end</a>
<li> <a href=work_states.html>Workunit and result states</a>
<li> <a href=tools_work.html>Generating work</a>
<li> <a href=retry.html>Generating result retries</a>
<li> <a href=validation.html>Result validation and accounting</a>
<li> <a href=result_process.html>Result processing</a>
<li> <a href=garbage.html>Garbage collection</a>
<li> <a href=result_process.html>Assimilation</a>
<li> <a href=garbage.html>File deletion</a>
<li> <a href=back_end.html>Back end examples</a>
</ul>

8
doc/index.html
View File

@ -8,10 +8,12 @@
<tr><td width=100%>
<center>
<font size=+1>
<b>
Berkeley Open Infrastructure for Network Computing
(BOINC)
is software that combines many volunteer PCs
into a parallel supercomputer.
</b>
</font>
</center>
</td></tr></table>
@ -74,6 +76,12 @@ width="210" height="62" border="0" alt="SourceForge Logo"></a>
<br>
Version 0.06 of BOINC is available on SourceForge.net.
<br><br>
<b>December 10, 2002</b>
<br>
We have started a beta test of BOINC using
the Astropulse application.
Many bugs have been found and fixed.
<br><br>
<b>November 25, 2002</b>
<br>
Version 0.03 of BOINC is available on SourceForge.net.

17
doc/intro.html
View File

@ -32,7 +32,7 @@ Existing applications in common languages (C, C++, Fortran)
can run as BOINC applications with little or no modification.
An application can consist of several files
(e.g. multiple programs and a coordinating script).
New versions of applications can be deployed with minimal participant involvement.
New versions of applications can be deployed with no participant involvement.
<li>
<b>Security</b>
<br>
@ -45,7 +45,14 @@ to protect against the distribution of viruses.
Projects can have separate scheduling and data servers,
with multiple servers of each type.
Clients automatically try alternate servers;
if all servers are unavailable, they do exponential backoff.
if all servers are down, clients do exponential backoff
to avoid flooding the servers when they come back up.
<li>
<b>System monitoring tools</b>
<br>
BOINC includes a web-based system for displaying time-varying
measurements (CPU load, network traffic, database table sizes).
This simplifies the task of diagnosing performance problems.
<li>
<b>Open source</b>
<br>
@ -80,7 +87,7 @@ Windows, Linux and other Unix systems).
The client can use multiple CPUs.
<li>
<b>Web-based administrative interfaces</b>
<b>Web-based participant interfaces</b>
<br>
BOINC provides web-based interfaces for
account creation, preference editing, and participant status display.
@ -92,6 +99,6 @@ making it easy to manage large numbers of hosts.
<br>
The core client downloads enough work to keep its host
busy for a user-specifiable amount of time.
This allows the host to continue to function during project downtimes,
and decreases the frequency of connections.
This can be used to decrease the frequency of connections or to
allow the host to keep working during project downtime.
</ul>

94
doc/project.fig
View File

@ -84,28 +84,6 @@ Single
0 106 #868286
0 107 #c7c3c7
0 108 #e7e3e7
6 3225 2625 4650 3300
2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
3225 2625 4650 2625 4650 3300 3225 3300 3225 2625
4 0 0 50 0 18 14 0.0000 4 210 1155 3375 2925 scheduling\001
4 0 0 50 0 18 14 0.0000 4 120 660 3600 3150 server\001
-6
6 4950 3675 5925 4950
5 1 0 1 -1 -1 0 0 -1 0.000 0 1 0 0 5400.000 4125.000 4950 4725 5400 4875 5850 4725
1 2 0 1 -1 11 0 0 20 0.000 1 0.0000 5421 3894 450 150 4971 3894 5871 3894
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 2
4950 3900 4950 4725
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 2
5850 3975 5850 4725
4 1 -1 0 0 18 12 0.0000 6 135 600 5400 4350 BOINC\001
4 1 -1 0 0 18 12 0.0000 6 135 270 5400 4575 DB\001
-6
6 7650 750 8775 1500
2 2 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 5
7650 750 8775 750 8775 1500 7650 1500 7650 750
4 0 0 50 0 18 14 0.0000 4 120 780 7800 1350 servers\001
4 0 0 50 0 18 14 0.0000 4 165 450 7950 1050 data\001
-6
6 2100 4050 2775 4800
2 1 0 1 0 3 50 0 20 0.000 0 0 -1 1 0 2
0 0 1.00 60.00 120.00
@ -118,24 +96,34 @@ Single
2400 5550 2400 6075
4 0 0 50 0 16 14 0.0000 4 165 510 2100 5475 client\001
-6
6 4800 2625 6225 3300
6 3225 4050 4650 4725
2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
4800 2625 6225 2625 6225 3300 4800 3300 4800 2625
4 0 0 50 0 18 14 0.0000 4 210 585 5175 2925 utility\001
4 0 0 50 0 18 14 0.0000 4 165 1005 5025 3150 programs\001
3225 4050 4650 4050 4650 4725 3225 4725 3225 4050
4 0 0 50 0 18 14 0.0000 4 210 1155 3375 4350 scheduling\001
4 0 0 50 0 18 14 0.0000 4 120 660 3600 4575 server\001
-6
6 6375 2625 7800 3300
6 6975 3975 8100 4725
2 2 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 5
6975 3975 8100 3975 8100 4725 6975 4725 6975 3975
4 0 0 50 0 18 14 0.0000 4 120 780 7125 4575 servers\001
4 0 0 50 0 18 14 0.0000 4 165 450 7275 4275 data\001
-6
6 8475 4050 9900 4725
2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
6375 2625 7800 2625 7800 3300 6375 3300 6375 2625
4 0 0 50 0 18 14 0.0000 4 165 465 6900 2925 Web\001
4 0 0 50 0 18 14 0.0000 4 165 1020 6600 3150 interfaces\001
8475 4050 9900 4050 9900 4725 8475 4725 8475 4050
4 0 0 50 0 18 14 0.0000 4 165 465 9000 4350 Web\001
4 0 0 50 0 18 14 0.0000 4 165 1020 8700 4575 interfaces\001
-6
6 5175 2175 6150 3450
5 1 0 1 -1 -1 0 0 -1 0.000 0 1 0 0 5625.000 2625.000 5175 3225 5625 3375 6075 3225
1 2 0 1 -1 11 0 0 20 0.000 1 0.0000 5646 2394 450 150 5196 2394 6096 2394
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 2
5175 2400 5175 3225
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 2
6075 2475 6075 3225
4 1 -1 0 0 18 12 0.0000 6 135 600 5625 2850 BOINC\001
4 1 -1 0 0 18 12 0.0000 6 135 270 5625 3075 DB\001
-6
2 2 0 1 0 30 50 0 20 0.000 0 0 -1 0 0 5
3600 750 6900 750 6900 1500 3600 1500 3600 750
2 1 0 1 0 3 50 0 20 0.000 0 0 -1 1 1 2
0 0 1.00 60.00 120.00
0 0 1.00 60.00 120.00
3900 3300 3900 7200
2 2 0 1 0 30 50 0 20 0.000 0 0 -1 0 0 5
9825 750 10350 750 10350 1200 9825 1200 9825 750
2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
@ -154,30 +142,34 @@ Single
3600 7200 8850 7200 8850 8100 3600 8100 3600 7200
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
0 0 1.00 60.00 120.00
8250 7200 8250 1500
7575 7200 7575 4725
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
1 1 1.00 60.00 120.00
4350 3300 4950 3750
0 0 1.00 60.00 120.00
9900 6225 9225 4725
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
1 1 1.00 60.00 120.00
5400 3300 5400 3675
0 0 1.00 60.00 120.00
3900 7200 3900 4725
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
1 1 1.00 60.00 120.00
6825 3300 6000 3750
0 0 1.00 60.00 120.00
4200 4050 5325 3300
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
1 1 1.00 60.00 120.00
6309 1488 7125 2625
0 0 1.00 60.00 120.00
8850 4050 6075 3225
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
1 1 1.00 60.00 120.00
5409 1488 5400 2625
0 0 1.00 60.00 120.00
5625 1650 5625 2250
2 2 0 1 0 11 50 0 20 0.000 0 0 -1 0 0 5
4350 1200 7650 1200 7650 1650 4350 1650 4350 1200
2 2 0 1 0 30 50 0 20 0.000 0 0 -1 0 0 5
4350 750 7650 750 7650 1200 4350 1200 4350 750
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 0 2
1 1 1.00 60.00 120.00
10081 6123 7500 3300
4 0 0 50 0 18 14 0.0000 4 210 1740 4350 1275 project back end\001
0 0 1.00 60.00 120.00
6825 1650 7500 4050
4 0 0 50 0 16 14 0.0000 4 210 2775 10500 1200 project-specific components\001
4 0 0 50 0 16 14 0.0000 4 210 1920 10500 1800 BOINC components\001
4 0 0 50 0 18 14 0.0000 4 210 1785 4751 6000 application client\001
4 0 0 50 0 18 14 0.0000 4 165 990 4981 6600 client API\001
4 0 0 50 0 18 14 0.0000 4 165 1095 5550 7725 core client\001
4 0 0 50 0 18 14 0.0000 4 210 2355 2100 2400 BOINC server complex\001
4 0 0 50 0 18 14 0.0000 4 210 1245 9450 6525 participants\001
4 0 0 50 0 18 14 0.0000 4 210 2355 2325 3675 BOINC server complex\001
4 0 0 50 0 18 14 0.0000 4 210 1740 2400 900 project back end\001

BIN
doc/project.png
View File

Binary file not shown.

80
doc/work_states.html Normal file
View File

@ -0,0 +1,80 @@
<title>Workunit and result states</title>
<h2>Workunit and result states</h2>
<p>
The processing of workunits and results involves
several independent activities.
To keep track of these activities,
workunit and result database records have several "state" fields,
and their processing can be viewed as the combination
of several finite-state machines.
<p>
A workunit has the following state fields:
<ul>
<li>
<b>main_state</b>:
Initially INIT.
If a canonical result is found (by validate)
it transitions to DONE.
If it's decided that no canonical result can be found
(by result_retry)
it transitions to ERROR.
<li>
<b>file_delete_state</b>:
Initially INIT.
When the main state transitions to either DONE or ERROR,
it transitions to READY,
indicating that input files can be deleted.
When file deletion is completed (by file_deleter)
it transitions to DONE.
<li>
<b>assimilate_state</b>:
Initially INIT.
When the main state transitions to either DONE or ERROR,
it transitions to READY,
indicating that the workunit can be assimilated.
When assimplateion is completed (by assimilator)
it transitions to DONE.
<li>
<b>need_validate</b>:
A boolean, true whenever
the workunit has a result whose validate state is NEED_CHECK.
The validate program sets it back to false.
</ul>
<p>
A result has the following state fields:
<ul>
<li>
<b>server_state</b>:
Initially UNSENT.
Becomes IN_PROGRESS when the result has been sent to a client.
Becomes DONE or ERROR if the client returns the result,
or TIMEOUT if result_retry gives up on the result.
<li>
<b>client_state</b>:
Records the client state (upload, process, or download)
where an error occurred.
Not relevant here.
<li>
<b>file_delete_state</b>:
Initially INIT.
When the parent workunit becomes ERROR, becomes READY,
indication that the output files can be deleted.
When the parent workunit becomes DONE, becomes READY
except for the canonical result, which remains INIT.
When the output files have been deleted (by file_deleter) becomes DONE.
<li>
<b>validate_state</b>:
Initially INITIAL.
When the result is returned, becomes NEED_CHECK.
When a canonical result has been found for the workunit,
becomes either VALID or INVALID.
</ul>

99
html/ops/db.inc
View File

@ -37,8 +37,8 @@ function show_platform($platform) {
start_table();
row("ID", $platform->id);
row("Created", time_str($platform->create_time));
row("Platform Name", $platform->name);
row("","<a href=db.php?show=app_version&plat_id=$platform->id>Application Versions for Platform $platform->name </a>");
row("Name", $platform->name);
row("","<a href=db.php?show=app_version&plat_id=$platform->id>App versions for this platform</a>");
end_table();
}
@ -46,9 +46,9 @@ function show_app($app) {
start_table();
row("ID", $app->id);
row("Created", time_str($app->create_time));
row("Application Name", $app->name);
row("","<a href=db.php?show=app_version&app_id=$app->id>Application Versions for Application $app->name</a>");
row("","<a href=db.php?show=workunit&app_id=$app->id>Workunits for this Application</a>");
row("Name", $app->name);
row("","<a href=db.php?show=app_version&app_id=$app->id>App Versions for this application</a>");
row("","<a href=db.php?show=workunit&app_id=$app->id>Workunits for this application</a>");
end_table();
}
@ -109,9 +109,17 @@ function show_host($host) {
}
function retry_state_str($s) {
function main_state_str($s) {
switch($s) {
case 0: return "Default";
case 0: return "Initial";
case 1: return "Done";
case 2: return "Error";
}
}
function error_str($s) {
switch($s) {
case 0: return "No error";
case 1: return "Send failed";
case 2: return "Too many errors (may have bug)";
case 3: return "Too many done (may be nondeterministic)";
@ -119,6 +127,22 @@ function retry_state_str($s) {
return "unknown";
}
function assimilate_state_str($s) {
switch($s) {
case 0: return "Initial";
case 1: return "Ready to assimilate";
case 2: return "Assimilated";
}
}
function file_delete_state_str($s) {
switch($s) {
case 0: return "Initial";
case 1: return "Ready to delete";
case 2: return "Deleted";
}
}
function show_workunit($wu,$show_xml_doc) {
start_table();
row("Created", time_str($wu->create_time));
@ -136,8 +160,11 @@ function show_workunit($wu,$show_xml_doc) {
row("Canonical resultid", $wu->canonical_resultid);
row("Canonical credit", $wu->canonical_credit);
row("Retry check time", time_str($wu->retry_check_time));
row("Retry state", retry_state_str($wu->state));
row("Delay bound", $wu->delay_bound);
row("Main state", main_state_str($wu->main_state));
row("Error", error_str($wu->error));
row("File delete state", file_delete_state_str($wu->file_delete_state));
row("Assimilation state", assimilate_state_str($wu->assimilate_state));
row("","<a href=db.php?show=result&wu_id=$wu->id>Show associated results</a>");
end_table();
echo "<p>";
@ -168,16 +195,16 @@ function validate_state_str($s) {
function show_result($result,$show_xml_docs,$show_stderr,$show_times) {
start_table();
if ($show_times) {
row("created", time_str($result->create_time));
row("sent", time_str($result->sent_time));
row("received", time_str($result->received_time));
row("Created", time_str($result->create_time));
row("Sent", time_str($result->sent_time));
row("Received", time_str($result->received_time));
}
row("Name", $result->name);
row("Workunit", "<a href=db.php?show=workunit&id=$result->workunitid>" . wu_name_by_id($result->workunitid) . "</a>" );
row("state", res_state_string($result->server_state));
row("Server state", res_state_string($result->server_state));
row("Deadline", time_str($result->report_deadline));
row("Host ID", "<a href=db.php?show=host&id=$result->hostid>" . host_name_by_id($result->hostid) . "</a>");
row("Client_state: ", $result->client_state);
row("Client state", $result->client_state);
row("CPU time", $result->cpu_time);
if ($show_xml_docs) {
row("XML doc in", "<pre>".htmlspecialchars($result->xml_doc_in)."</pre>");
@ -186,42 +213,42 @@ function show_result($result,$show_xml_docs,$show_stderr,$show_times) {
if ($show_stderr) {
row("stderr out", "<pre>".htmlspecialchars($result->stderr_out)."</pre>");
}
row("batch", $result->batch);
row("project state", $result->project_state);
row("Batch", $result->batch);
row("File delete state", file_delete_state_str($result->file_delete_state));
row("Validate state", validate_state_str($result->validate_state));
row("claimed credit", $result->claimed_credit);
row("granted credit", $result->granted_credit);
row("Granted credit", $result->granted_credit);
end_table();
echo "<p>";
}
function show_user($user) {
start_table();
row("created", time_str($user->create_time));
row("total credit", $user->total_credit);
row("recent averaged credit:", $user->expavg_credit);
row("name", $user->name);
row("email address", $user->email_addr);
row("country", $user->country);
row("postal code", $user->postal_code);
row("total credit", $user->total_credit);
row("average credit", $user->expavg_credit);
row("last average time", time_str($user->expavg_time));
row("Created", time_str($user->create_time));
row("Total credit", $user->total_credit);
row("Recent averaged credit:", $user->expavg_credit);
row("Name", $user->name);
row("Email address", $user->email_addr);
row("Country", $user->country);
row("Postal code", $user->postal_code);
row("Total credit", $user->total_credit);
row("Average credit", $user->expavg_credit);
row("Last average time", time_str($user->expavg_time));
end_table();
}
function team_type_string($s) {
switch ($s) {
case 1: return "Small Company";
case 2: return "Medium Company";
case 3: return "Large Company";
case 4: return "Club";
case 5: return "Primary School";
case 6: return "Secondary School";
case 7: return "Junior College";
case 8: return "University or Department";
case 9: return "Government Agency";
default: return "Unknown";
case 1: return "Small Company";
case 2: return "Medium Company";
case 3: return "Large Company";
case 4: return "Club";
case 5: return "Primary School";
case 6: return "Secondary School";
case 7: return "Junior College";
case 8: return "University or Department";
case 9: return "Government Agency";
default: return "Unknown";
}
}

17
html/user/user.inc
View File

@ -1,10 +1,11 @@
<?php
// show a user summary. Contains only public information
//
function show_user_stats($user) {
echo TABLE2."\n";
echo "<tr>".TD2.LG_FONT."<b>User Stats:</b></font></td></tr>\n";
row("<b>User Name: </b>", $user->name);
$row = sprintf("<b>%s user since: </b>", $project);
row("<b>Name: </b>", $user->name);
$row = sprintf("<b>%s Participant since: </b>", $project);
row($row, time_str($user->create_time));
row("<b>Total credit: </b>", $user->total_credit);
row("<b>Recent average credit: </b>", $user->expavg_credit);
@ -18,6 +19,8 @@ function show_user_stats($user) {
echo "</table>\n";
}
// show private user info
//
function show_user_profile($user) {
echo TABLE2."\n";
echo "<tr>".TD2.LG_FONT."<b>User Information:</b></font></td></tr>\n";
@ -48,10 +51,10 @@ function show_hosts($user) {
function show_host_brief($host) {
echo "<tr>
<td><a href=show_host_detail.php?hostid=$host->id>$host->domain_name</a></td>
<td>$host->total_credit</td>
<td>$host->expavg_credit</td>
</tr>";
<td><a href=show_host_detail.php?hostid=$host->id>$host->domain_name</a></td>
<td>$host->total_credit</td>
<td>$host->expavg_credit</td>
</tr>";
}
// show a summary of the user.

21
sched/assimilator.C
View File

@ -10,14 +10,15 @@ CONFIG config;
// return nonzero if did anything
//
bool do_pass() {
bool do_pass(APP app) {
WORKUNIT wu;
RESULT result;
bool did_something = false;
int retval;
wu.appid = app.id;
wu.assimilate_state = ASSIMILATE_READY;
while (db_workunit_enum_assimilate_state(wu)) {
while (!db_workunit_enum_app_assimilate_state(wu)) {
did_something = true;
switch(wu.main_state) {
case WU_MAIN_STATE_INIT:
@ -50,6 +51,7 @@ bool do_pass() {
int main(int argc, char** argv) {
int retval;
bool asynch = false, one_pass = false;
APP app;
int i;
for (i=1; i<argc; i++) {
@ -57,6 +59,8 @@ int main(int argc, char** argv) {
asynch = true;
} else if (!strcmp(argv[i], "-one_pass")) {
one_pass = true;
} else if (!strcmp(argv[i], "-app")) {
strcpy(app.name, argv[++i]);
} else {
fprintf(stderr, "Unrecognized arg: %s\n", argv[i]);
}
@ -75,11 +79,20 @@ int main(int argc, char** argv) {
}
retval = db_open(config.db_name, config.db_passwd);
if (retval) {
fprintf(stderr, "Can't open DB\n");
exit(1);
}
retval = db_app_lookup_name(app);
if (retval) {
fprintf(stderr, "Can't find app\n");
exit(1);
}
if (one_pass) {
do_pass();
do_pass(app);
} else {
while (1) {
if (!do_pass()) sleep(10);
if (!do_pass(app)) sleep(10);
}
}
}

12
sched/file_deleter.C
View File

@ -16,9 +16,7 @@ int wu_delete_files(WORKUNIT& wu) {
p = strtok(wu.xml_doc, "\n");
strcpy(filename, "");
while (p) {
p = strtok(0, "\n");
if (parse_str(p, "<name>", filename, sizeof(filename))) {
continue;
} else if (match_tag(p, "<file_info>")) {
no_delete = false;
strcpy(filename, "");
@ -27,9 +25,11 @@ int wu_delete_files(WORKUNIT& wu) {
} else if (match_tag(p, "</file_info>")) {
if (!no_delete) {
sprintf(pathname, "%s/%s", config.download_dir, filename);
printf("deleting %s\n", pathname);
unlink(pathname);
}
}
p = strtok(0, "\n");
}
return 0;
}
@ -41,9 +41,7 @@ int result_delete_files(RESULT& result) {
p = strtok(result.xml_doc_in, "\n");
while (p) {
p = strtok(0, "\n");
if (parse_str(p, "<name>", filename, sizeof(filename))) {
continue;
} else if (match_tag(p, "<file_info>")) {
no_delete = false;
strcpy(filename, "");
@ -52,9 +50,11 @@ int result_delete_files(RESULT& result) {
} else if (match_tag(p, "</file_info>")) {
if (!no_delete) {
sprintf(pathname, "%s/%s", config.upload_dir, filename);
printf("deleting %s\n", pathname);
unlink(pathname);
}
}
p = strtok(0, "\n");
}
return 0;
}
@ -67,7 +67,7 @@ bool do_pass() {
bool did_something = false;
wu.file_delete_state = FILE_DELETE_READY;
while (db_workunit_enum_file_delete_state(wu)) {
while (!db_workunit_enum_file_delete_state(wu)) {
did_something = true;
wu_delete_files(wu);
wu.file_delete_state = FILE_DELETE_DONE;
@ -75,7 +75,7 @@ bool do_pass() {
}
result.file_delete_state = FILE_DELETE_READY;
while (db_result_enum_file_delete_state(result)) {
while (!db_result_enum_file_delete_state(result)) {
did_something = true;
result_delete_files(result);
result.file_delete_state = FILE_DELETE_DONE;

3
sched/handle_request.C
View File

@ -363,9 +363,6 @@ int handle_results(
if (retval) {
fprintf(stderr, "Can't update WU\n");
}
fprintf(stderr,
"BOINC scheduler: set need_validate for WU %d\n", wu.id
);
}
}
}

14
test/test.inc
View File

@ -248,6 +248,8 @@ class Project {
PassThru("cp $source_dir/sched/feeder $this->project_dir/cgi/");
PassThru("cp $source_dir/sched/result_retry $this->project_dir/cgi/");
PassThru("cp $source_dir/sched/validate_test $this->project_dir/cgi/");
PassThru("cp $source_dir/sched/file_deleter $this->project_dir/cgi/");
PassThru("cp $source_dir/sched/assimilator $this->project_dir/cgi/");
PassThru("cp $source_dir/html_ops/stripchart.cgi $this->project_dir/cgi/");
$f = fopen("$this->project_dir/cgi/config.xml", "w");
fputs($f, "<config>\n");
@ -420,6 +422,18 @@ class Project {
PassThru("cd $this->project_dir/cgi; ./validate_test -one_pass -app $app->name -quorum $quorum > validate_out");
}
// do one pass of file_deleter
//
function file_delete() {
PassThru("cd $this->project_dir/cgi; ./file_deleter -one_pass > file_deleter_out");
}
// do one pass of assimilator
//
function assimilate($app) {
PassThru("cd $this->project_dir/cgi; ./assimilator -one_pass -app $app->name > assimilator_out");
}
// this should stop the feeder and any other daemons
//
function stop() {

5
test/test_uc.php
View File

@ -49,4 +49,9 @@
$project->check_results(2, $result);
$project->compare_file("uc_wu_0_0", "uc_correct_output");
$project->compare_file("uc_wu_1_0", "uc_correct_output");
$project->file_delete();
// input file should be gone here
$project->assimilate($app);
$project->file_delete();
?>

245
todo
View File

@ -1,36 +1,130 @@
write garbage collector
prevent 2 core clients from running at once
support for HTTP and SOCKS proxies
-----------------------
HIGH-PRIORITY (should do for beta test)
-----------------------
make get_local_ip_addr() work in all cases
est_time_to_completion doesn't work for non-running tasks
get timezone working on all platforms
get benchmark working as separate process
must do this before send first scheduler request
validate in crontab
run backend programs (validate/file_deleter/assimilate)
from crontab; document
Windows client
make mini-logo
fix "unable to calculate"
completed results: show final CPU time
Messages from core client
decide what messages should be shown to user, and how
log file? GUI? dialog?
-----------------------
MEDIUM-PRIORITY (should do before public release)
-----------------------
use https for login (don't sent account ID or password in clear)
protect project admin web pages (htaccess)
get timezone working on all platforms
Deadline mechanism for results
- use in result dispatching
- use in file uploading (decide what to upload next)
- use in deciding when to make scheduler RPC (done already?)
create "result adder" program that detects WUs that don't have
a canonical result yet, and should,
and creates more results for them.
This should detect situations where we're getting lots
of error results, and skip over those WUs
Testing framework
better mechanisms to model server/client/communication failure
better mechanisms to simulate large load
do client/server on separate hosts?
Delete files if needed to honor disk usage constraint
inform user if this happens
implement max bytes/sec network preferences
Global preferences
implement disk usage prefs
time-of-day prefs?
test propagation mechanism
set up multi-project, multi-host test;
change global prefs at one web site,
make sure they propagate to all hosts
limit on frequency of disk writes?
max net traffic per day?
implement in client
Per-project preferences
test project-specific prefs
make example web edit pages
make app that uses them
set up a test with multiple projects
test "add project" feature, GUI and cmdline
test resource share mechanism
CPU benchmarking
review CPU benchmarks - do they do what we want?
what to do when tests show hardware problem?
How should we weight factors for credit?
run CPU tests unobtrusively, periodically
check that on/conn/active fracs are maintainted correctly
check that bandwidth is measured correctly
measure disk/mem size on all platforms
get timezone to work
CPU accounting in the presence of checkpoint/restart
test
Test nslots > 1
Redundancy checking and validation
test the validation mechanism
make sure credit is granted correctly
make sure average, total credit maintained correctly for user, host
Windows screensaver functionality
idle-only behavior without screensaver - test
Data transfer
make sure restart of downloads works
make sure restart of uploads works
test download/upload with multiple data servers
make sure it tries servers in succession,
does exponential backoff if all fail
review and document prioritization of transfers
review protocol; make sure error returns are possible and handled correctly
Scheduler
Should dispatch results based on deadline?
test that scheduler estimates WU completion time correctly
test that scheduler sends right amount of work
test that client estimates remaining work correctly,
requests correct # of seconds
test that hi/low water mark system works
test that scheduler sends only feasible WUs
Scheduler RPC
formalize notion of "permanent failure" (e.g. can't download file)
report perm failures to scheduler, record in DB
make sure RPC backoff is done for any perm failure
(in general, should never make back-to-back RPCs to a project)
make sure that client eventually reloads master URL
Application graphics
finish design, implementation, doc, testing
size, frame rate, whether to generate
Work generation
generation of upload signature is very slow
prevent file_xfer->req1 from overflowing. This problems seems to be
happening when the file_upload_handler returnes a message to the
client that is large. This causes project->parsefile to get wrong
input and so on.
test HTTP redirect mechanism for all types of ops
Add batch features to ops web
-----------------------
LONG-TERM IDEAS AND PROJECTS
-----------------------
use https for login (don't sent account ID or password in clear)
CPU benchmarking
This should be done by a pseudo-application
@ -40,14 +134,6 @@ CPU benchmarking
benchmarking programs (e.g. for graphics coprocessor)
or project-specific programs.
Testing framework
better mechanisms to model server/client/communication failure
better mechanisms to simulate large load
do client/server on separate hosts?
Add "garbage-collect state" fields to WU, result
whether deletion of files is OK, is done
investigate binary diff mechanism for updating persistent files
verify support for > 4 GB files everywhere
@ -55,11 +141,6 @@ verify support for > 4 GB files everywhere
use FTP instead of HTTP for file xfer??
measure speed diff
Delete files if needed to honor disk usage constraint
inform user if this happens
implement max bytes/sec network preferences
Local scheduling
more intelligent decision about when/what to work on
- monitor VM situation, run small-footprint programs
@ -85,99 +166,12 @@ Include account ID in URL for file xfers
This would let you verify network xfers by scanning web logs
(could use that to give credit for xfers)
Global preferences
implement disk usage prefs
time-of-day prefs?
test propagation mechanism
set up multi-project, multi-host test;
change global prefs at one web site,
make sure they propagate to all hosts
limit on frequency of disk writes?
max net traffic per day?
implement in client
Per-project preferences
test project-specific prefs
make example web edit pages
make app that uses them
set up a test with multiple projects
test "add project" feature, GUI and cmdline
test resource share mechanism
Proxies
work through HTTP, Socks proxies
look at other open-source code (Mozilla?)
Documentation
simplify/finish docs on server installation
get rid of text in INSTALL and INSTALL_CLIENT;
they should just refer to .html files
Testing in general
figure out how to set up multi-project, multi-host tests
from a single script
automate some simple test cases
Messages from core client
decide what messages should be shown to user, and how
log file? GUI? dialog?
CPU benchmarking
review CPU benchmarks - do they do what we want?
what to do when tests show hardware problem?
How should we weight factors for credit?
run CPU tests unobtrusively, periodically
check that on/conn/active fracs are maintainted correctly
check that bandwidth is measured correctly
measure disk/mem size on all platforms
get timezone to work
WU/result sequence mechanism
design/implement/document
Multiple application files
document, test
CPU accounting in the presence of checkpoint/restart
test
Test nslots > 1
Redundancy checking and validation
test the validation mechanism
make sure credit is granted correctly
make sure average, total credit maintained correctly for user, host
Scheduler RPC
formalize notion of "permanent failure" (e.g. can't download file)
report perm failures to scheduler, record in DB
make sure RPC backoff is done for any perm failure
(in general, should never make back-to-back RPCs to a project)
make sure that client eventually reloads master URL
Data transfer
make sure restart of downloads works
make sure restart of uploads works
test download/upload with multiple data servers
make sure it tries servers in succession,
does exponential backoff if all fail
review and document prioritization of transfers
review protocol; make sure error returns are possible and handled correctly
Application graphics
finish design, implementation, doc, testing
size, frame rate, whether to generate
Work generation
generation of upload signature is very slow
Windows GUI
finish design/implement
display credit info, team
Windows screensaver functionality
idle-only behavior without screensaver - test
Versioning
think through issues involved in:
compatibility of core client and scheduling server
@ -187,29 +181,10 @@ Versioning
Need version numbers for protocols/interfaces?
What messages to show user? Project?
Scheduler
Should dispatch results based on deadline?
test that scheduler estimates WU completion time correctly
test that scheduler sends right amount of work
test that client estimates remaining work correctly,
requests correct # of seconds
test that hi/low water mark system works
test that scheduler sends only feasible WUs
Persistent files
test
design/implement test reporting, retrieval mechanisms
(do this using WU/results with null application?)
User HTML
leader boards
NET_XFER_SET
review logic; prevent one stream for starving others
test HTTP redirect mechanism for all types of ops
prevent file_xfer->req1 from overflowing. This problems seems to be
happening when the file_upload_handler returnes a message to the
client that is large. This causes project->parsefile to get wrong
input and so on.