2 #Last updated: May/14/2009 By Rami
3 #----------Standard / IPAR-----------------------
4 # Main Desc: Top level wrapper script for the whole Data pipeline after sequencing is done until data analysis.
5 # Start: Read current folder as input Run Folder (ex: 080220_HWI-FC2057U).
6 # This scripts assumes that:
7 # 1 - Sequence reading has finished and all data are in Run Folder.
8 # 2 - DataRun record saved with apropriate cnfig params in the FcTracker online DB (http://htsw.stanford.edu/admin)
9 # 3 - Curren directory, from were this script is ran, is the Run Folder
11 # START: (checks in ExpTrackLog dir if OKfiles exist before executing scripts)
12 # Get Input: <FlowCellId> ["init" (to start from scatch, otherwise starts after the last successful step)]
14 # Connect to the ExperimentInfo DB (FcTracker)
16 # Get LanesNames.xml file
17 # Aks the user to check if the retrieved parameters are OK. If Yes, enter "GO", otherwise <ENTER> and fix ..
19 # Runs the PipelineListener & for the rest of the process flow
20 # Goes down 2 dirs to Firecrest...etc .
21 # Runs the make -j ect ...
23 ################################################################################
27 runfolder=`pwd | sed -e 's/^.*\///'`
32 echo "Missing argument: FlowcellId. Try again. Example: ./MainPipelineScript FC99999 <enter>"
38 if [ $2 ] && [ $2 == 'init' ]
41 echo "==== Starting from scratch at `date` ===="
43 echo "=== Starting at `date` ===";
46 echo "=== Retrieving Information - Please wait ... ===";
50 else mkdir ExpTrackLog
53 exptrackmainsc="$EXPTRACKDIR/exp_track_main.py"
54 # -- Check Image folders and find out cycle number
55 tocycle=`$EXPTRACKDIR/CheckImageFolders`
57 ## Uncomment this if you want to force a cycle number:
60 if [ $tocycle -gt 0 ] && [ $tocycle -lt 37 ]
61 then echo "Cheked Imag Folders -> cycle set to: $tocycle"
63 echo "PROBLEM: Check Image folders failed to a cycle between 0 and 36. Found: $tocycle"
64 echo "Note: to input cycle number edit the MainP..Script un-comment ##tocycle=36"
65 echo " --- tocycle SET to 0 --- "
66 echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
73 analysis_mode='standard'
74 pipeline_version='1.3'
75 ## The Command that runs the making of the "make" files
76 if [ -d Data/IPAR_$pipeline_version ]
78 datadir=Data/IPAR_$pipeline_version
84 # control lane by default 4
85 makefilescmd="/Applications/SolexaPipeline/Goat/goat_pipeline.py --cycles=1-$tocycle --matrix=auto --phasing=auto --control-lane=4 --GERALD=config.txt $fullpath/$datadir --make"
86 if [ $analysis_mode == 'IPAR' ]
88 # For IPAR we can't use the goat. we use the bustard.py (see user man for Pipeline Version 1.3 page 22 at the bottom)
89 makefilescmd="/usr/local/src/GAPipeline-$pipeline_version.2/bin/bustard.py $fullpath/$datadir --make --cycles=1-$tocycle --matrix=auto --phasing=auto --control-lane=4 --GERALD=config.txt"
93 ## PRINT PRE CHECK REPORT
94 echo "=================================== PRELIMINARY CHECKS ================================"
96 echo "Config File generation"
98 python $exptrackmainsc getconfile $fcid $runfolder
101 echo "OK" > ExpTrackLog/OK_get_check_config_file
103 echo "ERROR: config.txt not found."
108 echo "------------------------------------------------------"
109 echo "LaneNames.xml file generation: "
112 python $exptrackmainsc getLanesNames $fcid
113 if [ -f LaneNames.xml ]
115 echo "LaneNames.xml found."
117 echo "ERROR: LaneNames.xml not found."
123 echo "----------------Analysis Params:----------------------"
124 echo "Analysis Mode: $analysis_mode"
125 echo "Pipeline Software Version: $pipeline_version"
126 echo "Cycles: 1 to $tocycle"
127 echo "Will run this to generate Make files: $makefilescmd"
128 echo "Full current path: $fullpath"
129 echo "Run Folder: $runfolder"
130 echo "FlowcellId: $fcid"
131 echo "------------------------------------------------------"
133 echo "NOTE!!: To change any of the above parameters just quit and modify this script according to your current needs."
136 echo "If these are the correct parameters for your Data Pipeline Run type GO to proceed"
138 echo "Otherwise type anything else to quit"
139 echo "====================================================================================="
141 if [ $2 ] && [ $2 == 'GO' ]
147 #######################
149 if [ $confirm ] && [ $confirm == "GO" ]
150 then echo "Proceeding .."
151 else echo "Response: "$confirm; exit
154 if [ -f ExpTrackLog/OK_get_check_config_file ]
156 echo "Ready for Make file generation step"
157 #run Make files generation step
158 python $exptrackmainsc updsts $fcid $runfolder 0 Generating_Make_files
161 echo "FAILED: get_check_config_file."
162 python $exptrackmainsc updsts $fcid $runfolder 0 Check_ConfigFile_Failed
166 ######################
168 ## Make sure we find the ONE specific Start folder (Firecrest or Standard / Bustard for IPAR), then check existance of "Makefile" these directories
169 if [ $analysis_mode == 'IPAR' ]
171 #firecrestdir=`ls $datadir | grep "Firecrest"`
172 #countdirs=`ls $datadir | grep -c Firecrest`
174 startdir=`ls $datadir | grep Bustard$pipeline_version`
175 countdirs=`ls $datadir | grep -c Bustard$pipeline_version`
177 startdir=`ls $datadir | grep C1-[0-9][0-9]_Firecrest`
178 countdirs=`ls $datadir | grep -c C1-[0-9][0-9]_Firecrest`
181 if [ $countdirs == 1 ]
183 echo "ok. start folder '$startdir' found"
185 if [ $countdirs -gt 1 ]
187 echo "ERROR: Found too many ($countdirs) start directories (firecrest/bustard(for IPAR)). You need to specify ONE folder or REMOVE duplicates. Found:"
190 python $exptrackmainsc updsts $fcid $runfolder 0 ERROR:_Need_Specific_Start_Dir
193 echo "ERROR: Start (firecrest/bustard(for IPAR)) directory NOT FOUND."
194 python $exptrackmainsc updsts $fcid $runfolder 0 ERROR:_Start_Dir_Not_Found
200 if [ $analysis_mode == 'IPAR' ]
202 firemakefile=$datadir/$startdir/Makefile # dummy check, since there's no need to check this one in IPAR
203 bustmakefile=$datadir/$startdir/Makefile
204 germakefile=$datadir/$startdir/GERALD_*/Makefile
206 firemakefile=$datadir/$startdir/Makefile
207 bustmakefile=$datadir/$startdir/Bustard*_*/Makefile
208 germakefile=$datadir/$startdir/Bustard*_*/GERALD_*/Makefile
211 #####################
214 if [ -f $firemakefile ] && [ -f $bustmakefile ] && [ -f $germakefile ]
216 #Launch MPListener in the background
217 $EXPTRACKDIR/MPListener_IPAR $fcid $runfolder $tocycle &
218 #Change dir down 2 folders to the Firecrest folder
219 echo "Changing dir to $datadir"
221 cd $datadir/$startdir
223 echo "Starting Solexa data process at `date`. This would be a good time for a climbing escapade at Lake Tahoe ... ;) ..."
224 python $exptrackmainsc updsts $fcid $runfolder 1
225 msgtext="Mode:$analysis_mode$pipeline_version"
226 python $exptrackmainsc updsts $fcid $runfolder 1 $msgtext
230 cp Bustard*/GERALD*/Summary.htm /Volumes/EncodeDisk4/$fcid_Summary.htm
234 echo "FAILED: can't find 3 Makefile. Looked for: $firemakefile, $bustmakefile, $germakefile"
236 python $exptrackmainsc updsts $fcid $runfolder 2 ERROR:_Make_files_not_found
240 ## MPListner takes care of the rest