$Fi fddlZddlZddlZddlZddlmZmZddlmZmZm Z m Z m Z m Z m Z mZmZddlmZddlmZddlmZddlmZddlmZd Zed zZd ZeZejd ZGd deZ ddZ!dZ"dZ#dZ$dZ%dZ&ddZ'dZ(dZ)dZ*ej+defdZ,dS)N)datetime timedelta) ColumnBooleanDateTimeIntegerString create_engineeventfunctext)declarative_base) Inspector)Session)close_all_sessions) DatabaseErrorz/var/lve/ssa.dbz.oldcleanup_old_dataceZdZdZdZeedZeeddZ eeddZ eedZ eedZ ee dZee dZeedZeedZee dZeedejd Zeedejej Zd S) RequestResultay Describes processed request stored in database file. E.g. { "timestamp": "1650008727", "url": "http://mydomain.com/index.php", "duration": 162077, "hitting_limits": false, "throttled_time": 0, "io_throttled_time": 0, "wordpress": true } Note: created_at, updated_at is saved in local TZ format scrape_resultT) primary_keyF)indexnullable)r)timezone)server_defaultr)onupdaterN)__name__ __module__ __qualname____doc__ __tablename__rridr domainpath timestampdurationris_slow_requesthitting_limitsthrottled_timeio_throttled_time wordpressrr now created_at updated_at=/opt/cloudlinux/venv/lib64/python3.11/site-packages/ssa/db.pyrr'sR $M T * * *B VF$ 7 7 7F 6&u 5 5 5Dw///Ivg...HfWu555OVGe444NVGe444Nw777w///I$/// RVWWWJ$///($(**U]UYU]U_U_```JJJr1r'ctjttz }d} t |5}|t d||d}|j}||z }|dkr dddn dddn #1swxYwYnt d|t|dS)a Removes outdated records from database, saving disk space. Deletions are batched to avoid loading the entire database into the cgroup's page cache, which causes OOM on large databases. Runs VACUUM after deletion to reclaim disk space. )daysrTzrDELETE FROM scrape_result WHERE rowid IN (SELECT rowid FROM scrape_result WHERE created_at < :cutoff LIMIT :batch))cutoffbatchNzCleanup deleted %d old records) rtodayrRETENTION_TIME_DAYS session_scopeexecuter rowcountloggerinfovacuum_database)engine batch_size n_days_ago total_deletedsessionresultdeleteds r2rrMs5!!I3F$G$G$GGJM  6 " " g__T\&&%z::<warningclose Exceptionerrorstr)r@raw_connoriginal_isolation_levelrLdb_dires r2r?r?ds$$&&H'7#' ""!8!899 9VRW % % b NNFVFFF G G G KKGvGG H H H H NN```` a a a 2333x    4555  111 /s1vv//000000001$< $< s0D-E F) F *F?F)F  F))Gcft|s!tj|dSdSN) is_db_presentBasemetadata create_all)r@s r2create_db_if_not_existrbs8  )   ((((())r1ctjtsdSt j|}d|D}t|dkS)NFcg|]}|Sr0r0).0tables r2 z!is_db_present..s G G Ge G G Gr1r)rMr%isfilerPr from_engineget_table_nameslen)r@database_inspectiontabless r2r^r^s^ 7>>& ! !u#/77 G G!4!D!D!F!F G G GF v;;?r1c0|ddS)NzPRAGMA journal_mode = WAL)r;) dbapi_con con_records r2setup_wal_moderqs 122222r1cdt|rdfd}td|d}tj|dtt ||S)Nzfile:z?mode=roc0tjdS)NT)uri)sqlite3connect)connection_stringsr2z!_setup_database..sgo&7TBBBr1 sqlite:////F)creatorechorv)rPr r listenrqrb)readonlyrzr@rws @r2_setup_databaser~sz(((;0:::BBBBG wUF LN3336""" Mr1Fc t|Sr])r~)r}s r2setup_databasers 8 $ $$r1ctjtr-t tjtt t}|tdt } t|5}t|5}| t t||dddn #1swxYwYdddn #1swxYwYn#t$rYnwxYwt tdztdzfD]5}tj|rtj|6dS)z Restore database by establish connections to old and new databases, merge data to new one if possible and delete old one. ryNz-walz-shm)rMr%existsrPrreplace OLD_SSA_DBrdisposer r:queryrfirst!merge_unharmed_data_from_databaserremove)r@ new_engine old_engine session_old session_newr%s r2restore_databasers  w~~f' 6:&&&!!J NN9Z99::J : & & H+}Z7P7P HT_   m , , 2 2 4 4 4 -k; G G G H H H H H H H H H H H H H H H H H H H H H H H H H H H H H H      Vf_fvo> 7>>$    IdOOOsZ DD +=C4( D 4C8 8D ;C8 <D ? D DDDD D$#D$cd}d} |t||} |}|sdS|D]}||nv#t $rit|D]V} |||z}|sn&||G#t $rYSwxYwYnwxYw| ||z })z_ Scrape all unharmed records from malformed database and merge them into new database. r TN) rroffsetlimitallmergerrangercommit) rrrrArrecords_to_saverecord pos_in_batchrecord_to_saves r2rrsNFJ!!-0077??EEjQQ #iikkO" ) * *!!&)))) *    %j 1 1   %*\\&<2G%H%H%N%N%P%PN)%%n5555$D  *%sHA:A::C-,CC-CC- C'$C-&C''C-,C-ctjtr=tjtrtjt t |5}|td}| }|dddkcdddS#1swxYwYdS#t$rYdSwxYw)z Try integrity check of database file to see if it is malformed. If database unable to execute it, will also count as malformed. zPRAGMA integrity_checkrokNT) rMr%rrrPrr:r;r fetchallr)r@dbrEerrorss r2is_malformed_databasers  w~~j!!bgnnV&<&< * 6 " " (bZZ%= > >??F__&&F!9Q<4' ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( tts7#C2AC: CC  CC C C"!C"returnc#Kt|} |V|n#|xYw |dS#|wxYw)zF Provide a transactional scope around a series of operations. )bindN)rrrollbackrT)r@rDs r2r:r:s 6"""G      s-AAAA4)r3)F)-rM contextlibruloggingrr sqlalchemyrrrrr r r r r sqlalchemy.ext.declarativersqlalchemy.engine.reflectionrsqlalchemy.ormrsqlalchemy.orm.sessionrsqlalchemy.excrrPrr9r_ getLoggerr=rrr?rbr^rqr~rrrrcontextmanagerr:r0r1r2rsN ((((((((877777222222""""""555555((((((  f_   - . .#a#a#a#a#aD#a#a#aL.&&&T))) 333   %%%%86      W      r1