class.jetpack-sync-module-full-sync.php 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366
  1. <?php
  2. /**
  3. * This class does a full resync of the database by
  4. * enqueuing an outbound action for every single object
  5. * that we care about.
  6. *
  7. * This class, and its related class Jetpack_Sync_Module, contain a few non-obvious optimisations that should be explained:
  8. * - we fire an action called jetpack_full_sync_start so that WPCOM can erase the contents of the cached database
  9. * - for each object type, we page through the object IDs and enqueue them by firing some monitored actions
  10. * - we load the full objects for those IDs in chunks of Jetpack_Sync_Module::ARRAY_CHUNK_SIZE (to reduce the number of MySQL calls)
  11. * - we fire a trigger for the entire array which the Jetpack_Sync_Listener then serializes and queues.
  12. */
  13. class Jetpack_Sync_Module_Full_Sync extends Jetpack_Sync_Module {
  14. const STATUS_OPTION_PREFIX = 'jetpack_sync_full_';
  15. const FULL_SYNC_TIMEOUT = 3600;
  16. public function name() {
  17. return 'full-sync';
  18. }
  19. function init_full_sync_listeners( $callable ) {
  20. // synthetic actions for full sync
  21. add_action( 'jetpack_full_sync_start', $callable );
  22. add_action( 'jetpack_full_sync_end', $callable );
  23. add_action( 'jetpack_full_sync_cancelled', $callable );
  24. }
  25. function init_before_send() {
  26. // this is triggered after actions have been processed on the server
  27. add_action( 'jetpack_sync_processed_actions', array( $this, 'update_sent_progress_action' ) );
  28. }
  29. function start( $module_configs = null ) {
  30. $was_already_running = $this->is_started() && ! $this->is_finished();
  31. // remove all evidence of previous full sync items and status
  32. $this->reset_data();
  33. if ( $was_already_running ) {
  34. /**
  35. * Fires when a full sync is cancelled.
  36. *
  37. * @since 4.2.0
  38. */
  39. do_action( 'jetpack_full_sync_cancelled' );
  40. }
  41. $this->update_status_option( 'started', time() );
  42. $this->update_status_option( 'params', $module_configs );
  43. $enqueue_status = array();
  44. $full_sync_config = array();
  45. // default value is full sync
  46. if ( ! is_array( $module_configs ) ) {
  47. $module_configs = array();
  48. foreach ( Jetpack_Sync_Modules::get_modules() as $module ) {
  49. $module_configs[ $module->name() ] = true;
  50. }
  51. }
  52. // set default configuration, calculate totals, and save configuration if totals > 0
  53. foreach ( Jetpack_Sync_Modules::get_modules() as $module ) {
  54. $module_name = $module->name();
  55. $module_config = isset( $module_configs[ $module_name ] ) ? $module_configs[ $module_name ] : false;
  56. if ( ! $module_config ) {
  57. continue;
  58. }
  59. if ( 'users' === $module_name && 'initial' === $module_config ) {
  60. $module_config = $module->get_initial_sync_user_config();
  61. }
  62. $enqueue_status[ $module_name ] = false;
  63. $total_items = $module->estimate_full_sync_actions( $module_config );
  64. // if there's information to process, configure this module
  65. if ( ! is_null( $total_items ) && $total_items > 0 ) {
  66. $full_sync_config[ $module_name ] = $module_config;
  67. $enqueue_status[ $module_name ] = array(
  68. $total_items, // total
  69. 0, // queued
  70. false, // current state
  71. );
  72. }
  73. }
  74. $this->set_config( $full_sync_config );
  75. $this->set_enqueue_status( $enqueue_status );
  76. /**
  77. * Fires when a full sync begins. This action is serialized
  78. * and sent to the server so that it knows a full sync is coming.
  79. *
  80. * @since 4.2.0
  81. */
  82. do_action( 'jetpack_full_sync_start', $full_sync_config );
  83. $this->continue_enqueuing( $full_sync_config, $enqueue_status );
  84. return true;
  85. }
  86. function continue_enqueuing( $configs = null, $enqueue_status = null ) {
  87. if ( ! $this->is_started() || $this->get_status_option( 'queue_finished' ) ) {
  88. return;
  89. }
  90. // if full sync queue is full, don't enqueue more items
  91. $max_queue_size_full_sync = Jetpack_Sync_Settings::get_setting( 'max_queue_size_full_sync' );
  92. $full_sync_queue = new Jetpack_Sync_Queue( 'full_sync' );
  93. $available_queue_slots = $max_queue_size_full_sync - $full_sync_queue->size();
  94. if ( $available_queue_slots <= 0 ) {
  95. return;
  96. } else {
  97. $remaining_items_to_enqueue = min( Jetpack_Sync_Settings::get_setting( 'max_enqueue_full_sync' ), $available_queue_slots );
  98. }
  99. if ( ! $configs ) {
  100. $configs = $this->get_config();
  101. }
  102. if ( ! $enqueue_status ) {
  103. $enqueue_status = $this->get_enqueue_status();
  104. }
  105. foreach ( Jetpack_Sync_Modules::get_modules() as $module ) {
  106. $module_name = $module->name();
  107. // skip module if not configured for this sync or module is done
  108. if ( ! isset( $configs[ $module_name ] )
  109. || // no module config
  110. ! $configs[ $module_name ]
  111. || // no enqueue status
  112. ! $enqueue_status[ $module_name ]
  113. || // finished enqueuing this module
  114. true === $enqueue_status[ $module_name ][ 2 ] ) {
  115. continue;
  116. }
  117. list( $items_enqueued, $next_enqueue_state ) = $module->enqueue_full_sync_actions( $configs[ $module_name ], $remaining_items_to_enqueue, $enqueue_status[ $module_name ][ 2 ] );
  118. $enqueue_status[ $module_name ][ 2 ] = $next_enqueue_state;
  119. // if items were processed, subtract them from the limit
  120. if ( ! is_null( $items_enqueued ) && $items_enqueued > 0 ) {
  121. $enqueue_status[ $module_name ][ 1 ] += $items_enqueued;
  122. $remaining_items_to_enqueue -= $items_enqueued;
  123. }
  124. // stop processing if we've reached our limit of items to enqueue
  125. if ( 0 >= $remaining_items_to_enqueue ) {
  126. $this->set_enqueue_status( $enqueue_status );
  127. return;
  128. }
  129. }
  130. $this->set_enqueue_status( $enqueue_status );
  131. // setting autoload to true means that it's faster to check whether we should continue enqueuing
  132. $this->update_status_option( 'queue_finished', time(), true );
  133. /**
  134. * Fires when a full sync ends. This action is serialized
  135. * and sent to the server.
  136. *
  137. * @since 4.2.0
  138. */
  139. do_action( 'jetpack_full_sync_end', '' );
  140. }
  141. function update_sent_progress_action( $actions ) {
  142. // quick way to map to first items with an array of arrays
  143. $actions_with_counts = array_count_values( array_filter( array_map( array( $this, 'get_action_name' ), $actions ) ) );
  144. if ( ! $this->is_started() || $this->is_finished() ) {
  145. return;
  146. }
  147. if ( isset( $actions_with_counts['jetpack_full_sync_start'] ) ) {
  148. $this->update_status_option( 'send_started', time() );
  149. }
  150. foreach ( Jetpack_Sync_Modules::get_modules() as $module ) {
  151. $module_actions = $module->get_full_sync_actions();
  152. $status_option_name = "{$module->name()}_sent";
  153. $items_sent = $this->get_status_option( $status_option_name, 0 );
  154. foreach ( $module_actions as $module_action ) {
  155. if ( isset( $actions_with_counts[ $module_action ] ) ) {
  156. $items_sent += $actions_with_counts[ $module_action ];
  157. }
  158. }
  159. if ( $items_sent > 0 ) {
  160. $this->update_status_option( $status_option_name, $items_sent );
  161. }
  162. }
  163. if ( isset( $actions_with_counts['jetpack_full_sync_end'] ) ) {
  164. $this->update_status_option( 'finished', time() );
  165. }
  166. }
  167. public function get_action_name( $queue_item ) {
  168. if ( is_array( $queue_item ) && isset( $queue_item[0] ) ) {
  169. return $queue_item[0];
  170. }
  171. return false;
  172. }
  173. public function is_started() {
  174. return !! $this->get_status_option( 'started' );
  175. }
  176. public function is_finished() {
  177. return !! $this->get_status_option( 'finished' );
  178. }
  179. public function get_status() {
  180. $status = array(
  181. 'started' => $this->get_status_option( 'started' ),
  182. 'queue_finished' => $this->get_status_option( 'queue_finished' ),
  183. 'send_started' => $this->get_status_option( 'send_started' ),
  184. 'finished' => $this->get_status_option( 'finished' ),
  185. 'sent' => array(),
  186. 'queue' => array(),
  187. 'config' => $this->get_status_option( 'params' ),
  188. 'total' => array(),
  189. );
  190. $enqueue_status = $this->get_enqueue_status();
  191. foreach ( Jetpack_Sync_Modules::get_modules() as $module ) {
  192. $name = $module->name();
  193. if ( ! isset( $enqueue_status[ $name ] ) ) {
  194. continue;
  195. }
  196. list( $total, $queued, $state ) = $enqueue_status[ $name ];
  197. if ( $total ) {
  198. $status[ 'total' ][ $name ] = $total;
  199. }
  200. if ( $queued ) {
  201. $status[ 'queue' ][ $name ] = $queued;
  202. }
  203. if ( $sent = $this->get_status_option( "{$name}_sent" ) ) {
  204. $status[ 'sent' ][ $name ] = $sent;
  205. }
  206. }
  207. return $status;
  208. }
  209. public function clear_status() {
  210. $prefix = self::STATUS_OPTION_PREFIX;
  211. Jetpack_Options::delete_raw_option( "{$prefix}_started" );
  212. Jetpack_Options::delete_raw_option( "{$prefix}_params" );
  213. Jetpack_Options::delete_raw_option( "{$prefix}_queue_finished" );
  214. Jetpack_Options::delete_raw_option( "{$prefix}_send_started" );
  215. Jetpack_Options::delete_raw_option( "{$prefix}_finished" );
  216. $this->delete_enqueue_status();
  217. foreach ( Jetpack_Sync_Modules::get_modules() as $module ) {
  218. Jetpack_Options::delete_raw_option( "{$prefix}_{$module->name()}_sent" );
  219. }
  220. }
  221. public function reset_data() {
  222. $this->clear_status();
  223. $this->delete_config();
  224. require_once dirname( __FILE__ ) . '/class.jetpack-sync-listener.php';
  225. $listener = Jetpack_Sync_Listener::get_instance();
  226. $listener->get_full_sync_queue()->reset();
  227. }
  228. private function get_status_option( $name, $default = null ) {
  229. $value = Jetpack_Options::get_raw_option( self::STATUS_OPTION_PREFIX . "_$name", $default );
  230. return is_numeric( $value ) ? intval( $value ) : $value;
  231. }
  232. private function update_status_option( $name, $value, $autoload = false ) {
  233. Jetpack_Options::update_raw_option( self::STATUS_OPTION_PREFIX . "_$name", $value, $autoload );
  234. }
  235. private function set_enqueue_status( $new_status ) {
  236. Jetpack_Options::update_raw_option( 'jetpack_sync_full_enqueue_status', $new_status );
  237. }
  238. private function delete_enqueue_status() {
  239. return Jetpack_Options::delete_raw_option( 'jetpack_sync_full_enqueue_status' );
  240. }
  241. private function get_enqueue_status() {
  242. return Jetpack_Options::get_raw_option( 'jetpack_sync_full_enqueue_status' );
  243. }
  244. private function set_config( $config ) {
  245. Jetpack_Options::update_raw_option( 'jetpack_sync_full_config', $config );
  246. }
  247. private function delete_config() {
  248. return Jetpack_Options::delete_raw_option( 'jetpack_sync_full_config' );
  249. }
  250. private function get_config() {
  251. return Jetpack_Options::get_raw_option( 'jetpack_sync_full_config' );
  252. }
  253. private function write_option( $name, $value ) {
  254. // we write our own option updating code to bypass filters/caching/etc on set_option/get_option
  255. global $wpdb;
  256. $serialized_value = maybe_serialize( $value );
  257. // try updating, if no update then insert
  258. // TODO: try to deal with the fact that unchanged values can return updated_num = 0
  259. // below we used "insert ignore" to at least suppress the resulting error
  260. $updated_num = $wpdb->query(
  261. $wpdb->prepare(
  262. "UPDATE $wpdb->options SET option_value = %s WHERE option_name = %s",
  263. $serialized_value,
  264. $name
  265. )
  266. );
  267. if ( ! $updated_num ) {
  268. $updated_num = $wpdb->query(
  269. $wpdb->prepare(
  270. "INSERT IGNORE INTO $wpdb->options ( option_name, option_value, autoload ) VALUES ( %s, %s, 'no' )",
  271. $name,
  272. $serialized_value
  273. )
  274. );
  275. }
  276. return $updated_num;
  277. }
  278. private function read_option( $name, $default = null ) {
  279. global $wpdb;
  280. $value = $wpdb->get_var(
  281. $wpdb->prepare(
  282. "SELECT option_value FROM $wpdb->options WHERE option_name = %s LIMIT 1",
  283. $name
  284. )
  285. );
  286. $value = maybe_unserialize( $value );
  287. if ( $value === null && $default !== null ) {
  288. return $default;
  289. }
  290. return $value;
  291. }
  292. }