Merge 10.6 into 10.7

cmake/install_macros.cmake

@@ -266,7 +266,7 @@ SET(DEBUGBUILDDIR "${BINARY_PARENTDIR}/debug" CACHE INTERNAL "Directory of debug
 FUNCTION(INSTALL_MYSQL_TEST from to)
   IF(INSTALL_MYSQLTESTDIR)
     IF(NOT WITH_WSREP)
-      SET(EXCL_GALERA "(suite/(galera|wsrep|sys_vars/[rt]/(sysvars_)?wsrep).*|include/((w.*)?wsrep.*|.*galera.*)\\.inc|std_data/(galera|wsrep).*)")
+      SET(EXCL_GALERA "(suite/(galera|wsrep|sys_vars/[rt]/(sysvars_)?wsrep).*|std_data/(galera|wsrep).*)")
     ELSE()
       SET(EXCL_GALERA "^DOES_NOT_EXIST$")
     ENDIF()

extra/mariabackup/fil_cur.cc

@@ -129,6 +129,7 @@ xb_fil_cur_open(
 	in case of error */
 	cursor->buf = NULL;
 	cursor->node = NULL;
+	cursor->n_process_batch = 0;
 
 	cursor->space_id = node->space->id;
 
@@ -374,6 +375,8 @@ xb_fil_cur_result_t xb_fil_cur_read(xb_fil_cur_t*	cursor,
 		return(XB_FIL_CUR_EOF);
 	}
 
+reinit_buf:
+	cursor->n_process_batch++;
 	if (to_read > (ib_int64_t) cursor->buf_size) {
 		to_read = (ib_int64_t) cursor->buf_size;
 	}
@@ -416,10 +419,28 @@ xb_fil_cur_result_t xb_fil_cur_read(xb_fil_cur_t*	cursor,
 
 	if (os_file_read(IORequestRead, cursor->file, cursor->buf, offset,
 			 (ulint) to_read, nullptr) != DB_SUCCESS) {
+		if (!srv_is_undo_tablespace(cursor->space_id)) {
 			ret = XB_FIL_CUR_ERROR;
 			goto func_exit;
 		}
 
+		if (cursor->buf_page_no
+		    >= SRV_UNDO_TABLESPACE_SIZE_IN_PAGES) {
+			ret = XB_FIL_CUR_SKIP;
+			goto func_exit;
+		}
+
+		to_read = SRV_UNDO_TABLESPACE_SIZE_IN_PAGES * page_size;
+
+		if (cursor->n_process_batch > 1) {
+			ret = XB_FIL_CUR_ERROR;
+			goto func_exit;
+		}
+
+		space->release();
+		goto reinit_buf;
+	}
+
 	defer = UT_LIST_GET_FIRST(space->chain)->deferred;
 	/* check pages for corruption and re-read if necessary. i.e. in case of
 	partially written pages */

extra/mariabackup/fil_cur.h

@@ -58,6 +58,7 @@ struct xb_fil_cur_t {
 	uint		thread_n;	/*!< thread number for diagnostics */
 	uint32_t	space_id;	/*!< ID of tablespace */
 	uint32_t	space_size;	/*!< space size in pages */
+	uint32_t	n_process_batch;/*!< Number of batch processed */
 
 	/** @return whether this is not a file-per-table tablespace */
 	bool is_system() const

extra/mariabackup/xtrabackup.cc

@@ -245,6 +245,10 @@ long innobase_file_io_threads = 4;
 ulong innobase_read_io_threads = 4;
 ulong innobase_write_io_threads = 4;
 
+/** Store the failed read of undo tablespace ids. Protected by
+backup mutex */
+static std::set<uint32_t> fail_undo_ids;
+
 longlong innobase_page_size = (1LL << 14); /* 16KB */
 char*	innobase_buffer_pool_filename = NULL;
 
@@ -406,6 +410,10 @@ struct ddl_tracker_t {
 
 static ddl_tracker_t ddl_tracker;
 
+/** Store the space ids of truncated undo log tablespaces. Protected
+by recv_sys.mutex */
+static std::set<uint32_t> undo_trunc_ids;
+
 /** Stores the space ids of page0 INIT_PAGE redo records. It is
 used to indicate whether the given deferred tablespace can
 be reconstructed. */
@@ -920,6 +928,11 @@ static void backup_file_op_fail(uint32_t space_id, int type,
 	}
 }
 
+static void backup_undo_trunc(uint32_t space_id)
+{
+  undo_trunc_ids.insert(space_id);
+}
+
 /* Function to store the space id of page0 INIT_PAGE
 @param	space_id	space id which has page0 init page */
 static void backup_first_page_op(space_id_t space_id)
@@ -2852,15 +2865,27 @@ static my_bool xtrabackup_copy_datafile(fil_node_t *node, uint thread_n,
 	}
 
 	/* The main copy loop */
-	while ((res = xb_fil_cur_read(&cursor, corrupted_pages)) ==
+	while (1) {
-		XB_FIL_CUR_SUCCESS) {
+		res = xb_fil_cur_read(&cursor, corrupted_pages);
+		if (res == XB_FIL_CUR_ERROR) {
+		       goto error;
+		}
+
+		if (res == XB_FIL_CUR_EOF) {
+			break;
+		}
+
 		if (!write_filter.process(&write_filt_ctxt, dstfile)) {
 			goto error;
 		}
-	}
 
-	if (res == XB_FIL_CUR_ERROR) {
+		if (res == XB_FIL_CUR_SKIP) {
-		goto error;
+			pthread_mutex_lock(&backup_mutex);
+			fail_undo_ids.insert(
+				static_cast<uint32_t>(cursor.space_id));
+			pthread_mutex_unlock(&backup_mutex);
+			break;
+		}
 	}
 
 	if (write_filter.finalize
@@ -4452,6 +4477,23 @@ static bool xtrabackup_backup_low()
 
 	dst_log_file = NULL;
 
+	std::vector<uint32_t> failed_ids;
+	std::set_difference(
+		fail_undo_ids.begin(), fail_undo_ids.end(),
+		undo_trunc_ids.begin(), undo_trunc_ids.end(),
+		std::inserter(failed_ids, failed_ids.begin()));
+
+	for (uint32_t id : failed_ids) {
+		msg("mariabackup: Failed to read undo log "
+		    "tablespace space id %d and there is no undo "
+		    "tablespace truncation redo record.",
+		    id);
+	}
+
+	if (failed_ids.size() > 0) {
+		return false;
+	}
+
 	if(!xtrabackup_incremental) {
 		strcpy(metadata_type, "full-backuped");
 		metadata_from_lsn = 0;
@@ -4526,6 +4568,7 @@ static bool xtrabackup_backup_func()
 
 	srv_operation = SRV_OPERATION_BACKUP;
 	log_file_op = backup_file_op;
+	undo_space_trunc = backup_undo_trunc;
 	first_page_init = backup_first_page_op;
 	metadata_to_lsn = 0;
 
@@ -4540,6 +4583,7 @@ static bool xtrabackup_backup_func()
 		}
 
 		log_file_op = NULL;
+		undo_space_trunc = NULL;
 		first_page_init = NULL;
 		if (dst_log_file) {
 			ds_close(dst_log_file);
@@ -4829,6 +4873,7 @@ static bool xtrabackup_backup_func()
 
 	innodb_shutdown();
 	log_file_op = NULL;
+	undo_space_trunc = NULL;
 	first_page_init = NULL;
 	pthread_mutex_destroy(&backup_mutex);
 	pthread_cond_destroy(&scanned_lsn_cond);

include/CMakeLists.txt

@@ -77,7 +77,6 @@ FOREACH(f ${HEADERS_GEN_CONFIGURE})
 ENDFOREACH(f)
 IF(NOT WITH_WSREP)
   SET(EXCL_SERVICE_WSREP "service_wsrep.h")
-  SET(EXCL_WSREP "wsrep.h")
 ENDIF()
 INSTALL(DIRECTORY mysql/
         DESTINATION ${INSTALL_INCLUDEDIR}/server/mysql COMPONENT Development
@@ -94,7 +93,6 @@ MACRO(INSTALL_PRIVATE DIR)
     FILES_MATCHING PATTERN "*.h"
     PATTERN CMakeFiles EXCLUDE
     PATTERN mysql EXCLUDE
-    PATTERN "${EXCL_WSREP}" EXCLUDE
     REGEX "\\./(${EXCL_RE}$)" EXCLUDE)
 ENDMACRO()
 

mysql-test/main/group_min_max.result

@@ -4066,6 +4066,26 @@ owner_id
 1
 DROP TABLE t1;
 #
+# MDEV-30240 Wrong result upon aggregate function with SQL_BUFFER_RESULT
+#
+drop table if exists t1,t2;
+Warnings:
+Note	1051	Unknown table 'test.t1,test.t2'
+CREATE TABLE t1 (pk INT PRIMARY KEY);
+INSERT INTO t1 VALUES (1),(2);
+CREATE TABLE t2 (a INT);
+INSERT INTO t2 VALUES (1),(2);
+SELECT SQL_BUFFER_RESULT MIN(pk) FROM t1, t2;
+MIN(pk)
+1
+SELECT MIN(pk) FROM t1, t2;
+MIN(pk)
+1
+DROP TABLE t1, t2;
+#
+# End of 10.5 tests
+#
+#
 # MDEV-24353: Adding GROUP BY slows down a query
 #
 CREATE TABLE t1 (p int NOT NULL, a int NOT NULL, PRIMARY KEY (p,a));

mysql-test/main/group_min_max.test

@@ -1725,6 +1725,23 @@ SELECT DISTINCT owner_id FROM t1 WHERE foo = true GROUP BY owner_id HAVING (COUN
 SELECT DISTINCT owner_id FROM t1 WHERE foo = true GROUP BY owner_id HAVING (COUNT(*) = 1);
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-30240 Wrong result upon aggregate function with SQL_BUFFER_RESULT
+--echo #
+
+drop table if exists t1,t2;
+CREATE TABLE t1 (pk INT PRIMARY KEY);
+INSERT INTO t1 VALUES (1),(2);
+CREATE TABLE t2 (a INT);
+INSERT INTO t2 VALUES (1),(2);
+SELECT SQL_BUFFER_RESULT MIN(pk) FROM t1, t2;
+SELECT MIN(pk) FROM t1, t2;
+DROP TABLE t1, t2;
+
+--echo #
+--echo # End of 10.5 tests
+--echo #
+
 --echo #
 --echo # MDEV-24353: Adding GROUP BY slows down a query
 --echo #

mysql-test/main/range_aria_dbt3.result

@@ -0,0 +1,13 @@
+set default_storage_engine=Aria;
+CREATE DATABASE dbt3_s001;
+use dbt3_s001;
+#
+# MDEV-30325 Wrong result upon range query using index condition
+#
+SELECT COUNT(*) FROM lineitem force index (i_l_orderkey_quantity,i_l_shipdate) WHERE l_shipdate < '1994-01-01' AND l_orderkey < 800 OR l_quantity > 3 AND l_orderkey NOT IN ( 157, 1444 );
+COUNT(*)
+5056
+#
+# End of 10.5 tests
+#
+DROP DATABASE dbt3_s001;

mysql-test/main/range_aria_dbt3.test

@@ -0,0 +1,24 @@
+#
+# This is generic tests using dbt3_s001 tables
+# This file uses the Aria storage engine
+#
+
+set default_storage_engine=Aria;
+
+CREATE DATABASE dbt3_s001;
+use dbt3_s001;
+--disable_query_log
+--source include/dbt3_s001.inc
+--enable_query_log
+
+--echo #
+--echo # MDEV-30325 Wrong result upon range query using index condition
+--echo #
+
+SELECT COUNT(*) FROM lineitem force index (i_l_orderkey_quantity,i_l_shipdate) WHERE l_shipdate < '1994-01-01' AND l_orderkey < 800 OR l_quantity > 3 AND l_orderkey NOT IN ( 157, 1444 );
+
+--echo #
+--echo # End of 10.5 tests
+--echo #
+
+DROP DATABASE dbt3_s001;

mysql-test/suite/innodb_fts/r/misc_debug.result

@@ -68,7 +68,7 @@ DROP TABLE t1;
 CREATE TABLE t1(f1 INT NOT NULL, f2 CHAR(100))ENGINE=InnoDB;
 SET DEBUG_DBUG="+d,stats_lock_fail";
 ALTER TABLE t1 ADD FULLTEXT(f2);
-ERROR HY000: Got error 15 "Block device required" from storage engine InnoDB
+ERROR HY000: Lock wait timeout exceeded; try restarting transaction
 SET debug_dbug=@saved_debug_dbug;
 ALTER TABLE t1 DISCARD TABLESPACE;
 ALTER TABLE t1 ADD FULLTEXT(f2);

mysql-test/suite/innodb_fts/t/misc_debug.test

@@ -108,7 +108,7 @@ DROP TABLE t1;
 
 CREATE TABLE t1(f1 INT NOT NULL, f2 CHAR(100))ENGINE=InnoDB;
 SET DEBUG_DBUG="+d,stats_lock_fail";
---error ER_GET_ERRNO
+--error ER_LOCK_WAIT_TIMEOUT
 ALTER TABLE t1 ADD FULLTEXT(f2);
 SET debug_dbug=@saved_debug_dbug;
 ALTER TABLE t1 DISCARD TABLESPACE;

mysys/array.c

@@ -23,6 +23,7 @@
 
   SYNOPSIS
     init_dynamic_array2()
+      ps_key            Key to register instrumented memory
       array		Pointer to an array
       element_size	Size of element
       init_buffer       Initial buffer pointer

mysys/hash.c

@@ -60,8 +60,9 @@ my_hash_value_type my_hash_sort(CHARSET_INFO *cs, const uchar *key,
   dynamic array that is part of the hash will allocate memory
   as required during insertion.
 
+  @param[in]     psi_key      The key to register instrumented memory
   @param[in,out] hash         The hash that is initialized
-  @param[in[     growth_size  size incrememnt for the underlying dynarray
+  @param[in]     growth_size  size incrememnt for the underlying dynarray
   @param[in]     charset      The character set information
   @param[in]     size         The hash size
   @param[in]     key_offest   The key offset for the hash

mysys/my_alloc.c

@@ -120,8 +120,8 @@ static void calculate_block_sizes(MEM_ROOT *mem_root, size_t block_size,
 
   SYNOPSIS
     init_alloc_root()
+      key            - key to register instrumented memory
       mem_root       - memory root to initialize
-      name           - name of memroot (for debugging)
       block_size     - size of chunks (blocks) used for memory allocation.
                        Will be updated to next power of 2, minus
                        internal and system memory management size.  This is

mysys/my_malloc.c

@@ -59,6 +59,7 @@ void set_malloc_size_cb(MALLOC_SIZE_CB func)
 /**
   Allocate a sized block of memory.
 
+  @param key    Key to register instrumented memory
   @param size   The size of the memory block in bytes.
   @param flags  Failure action modifiers (bitmasks).
 
@@ -120,6 +121,7 @@ void *my_malloc(PSI_memory_key key, size_t size, myf my_flags)
 /**
    @brief wrapper around realloc()
 
+   @param  key             key to register instrumented memory
    @param  old_point       pointer to currently allocated area
    @param  size            new size requested, must be >0
    @param  my_flags        flags

sql/CMakeLists.txt

@@ -508,7 +508,7 @@ IF(WIN32)
 ENDIF(WIN32)
 
 IF(NOT WITH_WSREP)
-  SET(EXCL_WSREP "wsrep*.h")
+  SET(EXCL_WSREP "wsrep_[a-np-z]*.h")
 ENDIF()
 INSTALL(DIRECTORY . DESTINATION ${INSTALL_INCLUDEDIR}/server/private COMPONENT Development
   FILES_MATCHING PATTERN "*.h"

sql/item_sum.cc

@@ -2373,8 +2373,15 @@ Item *Item_sum_variance::result_item(THD *thd, Field *field)
 void Item_sum_min_max::clear()
 {
   DBUG_ENTER("Item_sum_min_max::clear");
+  /*
+    We should not clear const items (from SELECT MIN(key) from t1) as then we would loose the
+    value cached in opt_sum_query() where we replace MIN/MAX/COUNT with constants.
+  */
+  if (!const_item())
+  {
     value->clear();
     null_value= 1;
+  }
   DBUG_VOID_RETURN;
 }
 
@@ -2487,9 +2494,12 @@ void Item_sum_min_max::no_rows_in_result()
   /* We may be called here twice in case of ref field in function */
   if (was_values)
   {
+    bool org_const_item_cache= const_item_cache;
     was_values= FALSE;
     was_null_value= value->null_value;
+    const_item_cache= 0;             // Ensure that clear works on const items
     clear();
+    const_item_cache= org_const_item_cache;
   }
   DBUG_VOID_RETURN;
 }

sql/opt_range.cc

@@ -1890,7 +1890,7 @@ SEL_ARG::SEL_ARG(SEL_ARG &arg) :Sql_alloc()
   next= 0;
   if (next_key_part)
   {
-    ++next_key_part->use_count;
+    next_key_part->increment_use_count(1);
     weight += next_key_part->weight;
   }
 }
@@ -10605,8 +10605,7 @@ key_or(RANGE_OPT_PARAM *param, SEL_ARG *key1,SEL_ARG *key2)
             Move on to next range in key2
           */
           key2->increment_use_count(-1); // Free not used tree
-          key2=key2_next;
+          key2= key2_next;
-          continue;
         }
         else
         {
@@ -10620,8 +10619,9 @@ key_or(RANGE_OPT_PARAM *param, SEL_ARG *key1,SEL_ARG *key2)
             tmp:     [---------]
           */
           key2->copy_max_to_min(tmp);
-          continue;
+          key2= key2_next;
         }
+        continue;
       }
 
       /*

sql/opt_sum.cc

@@ -438,7 +438,7 @@ int opt_sum_query(THD *thd,
             The optimization is not applicable in both cases:
             (a) 'expr' is a non-constant expression. Then we can't
             replace 'expr' by a constant.
-            (b) 'expr' is a costant. According to ANSI, MIN/MAX must return
+            (b) 'expr' is a constant. According to ANSI, MIN/MAX must return
             NULL if the query does not return any rows. Thus, if we are not
             able to determine if the query returns any rows, we can't apply
             the optimization and replace MIN/MAX with a constant.

sql/sql_select.cc

@@ -26265,7 +26265,6 @@ bool JOIN::alloc_func_list()
   @param field_list        All items
   @param send_result_set_metadata       Items in select list
   @param before_group_by   Set to 1 if this is called before GROUP BY handling
-  @param recompute         Set to TRUE if sum_funcs must be recomputed
 
   @retval
     0  ok

storage/innobase/handler/handler0alter.cc

@@ -11273,7 +11273,7 @@ ha_innobase::commit_inplace_alter_table(
 	}
 
 	DBUG_EXECUTE_IF("stats_lock_fail",
-			error = DB_LOCK_WAIT;);
+			error = DB_LOCK_WAIT_TIMEOUT;);
 
 	if (error == DB_SUCCESS) {
 		error = lock_sys_tables(trx);

storage/innobase/include/log0recv.h

@@ -92,6 +92,11 @@ extern void (*log_file_op)(uint32_t space_id, int type,
 			   const byte* name, ulint len,
 			   const byte* new_name, ulint new_len);
 
+/** Report an operation which does undo log tablespace truncation
+during backup
+@param	space_id	undo tablespace identifier */
+extern void (*undo_space_trunc)(uint32_t space_id);
+
 /** Report an operation which does INIT_PAGE for page0 during backup.
 @param	space_id	tablespace identifier */
 extern void (*first_page_init)(uint32_t space_id);

storage/innobase/log/log0recv.cc

@@ -1037,6 +1037,8 @@ void (*log_file_op)(uint32_t space_id, int type,
 		    const byte* name, ulint len,
 		    const byte* new_name, ulint new_len);
 
+void (*undo_space_trunc)(uint32_t space_id);
+
 void (*first_page_init)(uint32_t space_id);
 
 /** Information about initializing page contents during redo log processing.
@@ -2434,6 +2436,8 @@ bool recv_sys_t::parse(lsn_t checkpoint_lsn, store_t *store, bool apply)
                         TRX_SYS_MAX_UNDO_SPACES, "compatibility");
           truncated_undo_spaces[space_id - srv_undo_space_id_start]=
             { recovered_lsn, page_no };
+          if (undo_space_trunc)
+            undo_space_trunc(space_id);
 #endif
           last_offset= 1; /* the next record must not be same_page  */
           continue;