The idea of the regular crawler of the scratch

scrapy Crawler timing setting
        Set a in the startup file while loop , Then create two files , A data storage crawler needs to continue crawling , A flag to determine whether the crawler is running .
        When running, judge whether the crawler runs the tag file 2 Whether there is ,
                If it doesn't exist , Use isExsit = os.path.isdir( file 1) Judge the continued file 1
                        If it exists, use shutil.rmtree Delete all files in the directory ,

                        If it doesn't exist, it will output no crawler
               cmdline Start the crawler file

       If the run file exists ： Output text file is running

        Every time 10 Second pause , Then set a variable to record the time , Once the time is exceeded , Just jump out while.

from scrapy import cmdline
import datetime
import time
import shutil
import os
# Crawler task timing settings 

# This is a directory created for crawlers to continue crawling . Continued data storage 
recoderDir = r"C:/Users/stawind/Desktop/spider/cninfospider1"
# Flag to determine whether the crawler is running 
checkFile = "C:/Users/stawind/Desktop/spider/isRunning.txt"

startTime = datetime.datetime.now()
print(f"startTime={startTime}")

i = 0
moniter = 0

while True:
    isRunning = os.path.isfile(checkFile)
    if not isRunning:
        # Deal with something before the crawler starts , Clean up jobdir = crawls
        isExsit = os.path.isdir(recoderDir)
        print(f"cninfospider not running,ready to start.isExsit:{isExsit}")
        if isExsit:
            # Delete continuation directory crawls And all files in the directory 
            removeRes = shutil.rmtree(recoderDir)
            print(f"At time:{datetime.datetime.now()}, delete res:{removeRes}")
        else:
            print(f"At time:{datetime.datetime.now()}, Dir:{recoderDir} is not exsit.")
        time.sleep(20)
        clawerTime = datetime.datetime.now()
        waitTime = clawerTime - startTime
        print(f"At time:{clawerTime}, start clawer: mySpider !!!, waitTime:{waitTime}")
        cmdline.execute('scrapy crawl cninfospider1 -s JOBDIR=C:/Users/stawind/Desktop/spider/cninfospider1/storeMyRequest'.split())
        break # Exit the script after the crawler 
    else:
        print(f"At time:{datetime.datetime.now()}, mySpider is running, sleep to wait.")
    i += 1

    time.sleep(10)
    moniter += 10
    if moniter >= 1440:
        break