🐾 Change the strategy

celseq2/celseq2.py

@@ -16,9 +16,9 @@ Select top-used parameters from snakemake.snakemake():
     * -p
     * -r
     * -T
-    * -w 1800 # 30min
-    * --keep-going
-    * --restart-times 1
+    * -w 300 # 5min
+    # * --keep-going (no longer True)
+    * --restart-times 2
 
 Select optional parameters from snakemake.snakemake():
     * --ri v.s. --ii
@@ -135,10 +135,10 @@ def main():
         printshellcmds=True,
         printreason=True,
         timestamp=True,
-        latency_wait=1800,
+        latency_wait=300,
         jobname="celseq2_job.{rulename}.{jobid}.sh",
-        keepgoing=True,
-        restart_times=1,
+        keepgoing=False,
+        restart_times=2,
 
         dryrun=args.dryrun,
         lock=not args.nolock,

celseq2/workflow/celseq2.snakemake

@@ -160,16 +160,20 @@ rule COUNT_MATRIX:
         touch('_done_UMI')
     message: 'Finished counting UMI-count matrix.'
     run:
-        if ALIGNER == 'star':
-            shell('rm {}'.format('_done_star_genome_loaded'))
-            print('Free memory loaded by STAR', flush=True)
-            with tempfile.TemporaryDirectory() as tmpdirname:
-                cmd = 'STAR '
-                cmd += '--genomeLoad Remove '
-                cmd += '--genomeDir {STAR_INDEX_DIR} '
-                cmd += '--outFileNamePrefix '
-                cmd += join_path(tmpdirname, '')
-                shell(cmd)
+        # if ALIGNER == 'star':
+        #     shell('rm {}'.format('_done_star_genome_loaded'))
+        #     print('Free memory loaded by STAR', flush=True)
+        #     with tempfile.TemporaryDirectory() as tmpdirname:
+        #         cmd = 'STAR '
+        #         cmd += '--genomeLoad Remove '
+        #         cmd += '--genomeDir {STAR_INDEX_DIR} '
+        #         cmd += '--outFileNamePrefix '
+        #         cmd += join_path(tmpdirname, '')
+        #         shell(cmd)
+        if glob.glob('celseq2_job*.sh*'):
+            mkfolder(SUBDIR_QSUB)
+            shell('mv -f celseq2_job*.sh* {}'.format(SUBDIR_QSUB))
+
         print_logger('UMI-count matrix is saved at {}'.format(input.csv))
 
 
@@ -181,6 +185,11 @@ if RUN_CELSEQ2_TO_ST:
         message: 'Convert to ST format.'
         output:
             touch('_done_ST')
+        run:
+            if glob.glob('celseq2_job*.sh*'):
+                mkfolder(SUBDIR_QSUB)
+                shell('mv -f celseq2_job*.sh* {}'.format(SUBDIR_QSUB))
+
 
     rule _celseq2_to_st:
         input:
@@ -304,21 +313,21 @@ if ALIGNER == 'bowtie2':
 
 
 if ALIGNER == 'star':
-    rule star_load_genome:
-        output:
-            flag = '_done_star_genome_loaded',
-        message: 'Loading genome to memory for STAR'
-        shadow: "shallow"
-        run:
-            cmd = 'STAR '
-            cmd += '--genomeLoad LoadAndExit '
-            cmd += '--genomeDir {STAR_INDEX_DIR} '
-            cmd += '&& echo loaded >>  {output.flag} '
-            shell(cmd)
+    # rule star_load_genome:
+    #     output:
+    #         flag = '_done_star_genome_loaded',
+    #     message: 'Loading genome to memory for STAR'
+    #     shadow: "shallow"
+    #     run:
+    #         cmd = 'STAR '
+    #         cmd += '--genomeLoad LoadAndExit '
+    #         cmd += '--genomeDir {STAR_INDEX_DIR} '
+    #         cmd += '&& echo loaded >>  {output.flag} '
+    #         shell(cmd)
 
     rule align_star:
         input:
-            flag = rules.star_load_genome.output.flag, #'_done_star_genome_loaded',
+            # flag = rules.star_load_genome.output.flag, #'_done_star_genome_loaded',
             fq = join_path(DIR_PROJ, SUBDIR_FASTQ, '{itemid}', '{bc}.fastq'),
         output:
             sam = join_path(DIR_PROJ, SUBDIR_ALIGN, '{itemid}', '{bc}.sam'),
@@ -335,7 +344,7 @@ if ALIGNER == 'star':
         run:
             cmd = 'STAR '
             cmd += ' --runRNGseed 42 '
-            cmd += ' --genomeLoad LoadAndKeep '
+            cmd += ' --genomeLoad NoSharedMemory '
             cmd += ' --runThreadN {params.threads} '
             cmd += ' --genomeDir {STAR_INDEX_DIR} '
             # cmd += ' --readFilesCommand zcat '