Merge tag 'zfs-2.3.0-rc1'

OpenZFS 2.3.0-rc1

Signed-off-by: Ameer Hamza <[email protected]>

META

@@ -1,8 +1,8 @@
 Meta:          1
 Name:          zfs
 Branch:        1.0
-Version:       2.2.99
-Release:       1
+Version:       2.3.0
+Release:       rc1
 Release-Tags:  relext
 License:       CDDL
 Author:        OpenZFS

config/deb.am

@@ -101,3 +101,5 @@ native-deb-kmod: native-deb-local debian
 	fakeroot debian/rules override_dh_binary-modules;
 
 native-deb: native-deb-utils native-deb-kmod
+
+.NOTPARALLEL: native-deb native-deb-utils native-deb-kmod

contrib/Makefile.am

@@ -1,4 +1,5 @@
 include $(srcdir)/%D%/bash_completion.d/Makefile.am
+include $(srcdir)/%D%/debian/Makefile.am
 include $(srcdir)/%D%/pyzfs/Makefile.am
 include $(srcdir)/%D%/zcp/Makefile.am
 

contrib/debian/Makefile.am

@@ -0,0 +1,48 @@
+dist_noinst_DATA += %D%/changelog.in
+dist_noinst_DATA += %D%/clean
+dist_noinst_DATA += %D%/control
+dist_noinst_DATA += %D%/control.modules.in
+dist_noinst_DATA += %D%/copyright
+dist_noinst_DATA += %D%/Makefile.am
+dist_noinst_DATA += %D%/not-installed
+dist_noinst_DATA += %D%/openzfs-libnvpair3.docs
+dist_noinst_DATA += %D%/openzfs-libnvpair3.install.in
+dist_noinst_DATA += %D%/openzfs-libpam-zfs.install
+dist_noinst_DATA += %D%/openzfs-libpam-zfs.postinst
+dist_noinst_DATA += %D%/openzfs-libpam-zfs.prerm
+dist_noinst_DATA += %D%/openzfs-libuutil3.docs
+dist_noinst_DATA += %D%/openzfs-libuutil3.install.in
+dist_noinst_DATA += %D%/openzfs-libzfs4.docs
+dist_noinst_DATA += %D%/openzfs-libzfs4.install.in
+dist_noinst_DATA += %D%/openzfs-libzfsbootenv1.docs
+dist_noinst_DATA += %D%/openzfs-libzfsbootenv1.install.in
+dist_noinst_DATA += %D%/openzfs-libzfs-dev.docs
+dist_noinst_DATA += %D%/openzfs-libzfs-dev.install.in
+dist_noinst_DATA += %D%/openzfs-libzpool5.docs
+dist_noinst_DATA += %D%/openzfs-libzpool5.install.in
+dist_noinst_DATA += %D%/openzfs-python3-pyzfs.install
+dist_noinst_DATA += %D%/openzfs-zfs-dkms.config
+dist_noinst_DATA += %D%/openzfs-zfs-dkms.dkms
+dist_noinst_DATA += %D%/openzfs-zfs-dkms.docs
+dist_noinst_DATA += %D%/openzfs-zfs-dkms.install
+dist_noinst_DATA += %D%/openzfs-zfs-dkms.postinst
+dist_noinst_DATA += %D%/openzfs-zfs-dkms.prerm
+dist_noinst_DATA += %D%/openzfs-zfs-dkms.templates
+dist_noinst_DATA += %D%/openzfs-zfs-dkms.triggers
+dist_noinst_DATA += %D%/openzfs-zfs-dracut.install
+dist_noinst_DATA += %D%/openzfs-zfs-initramfs.install
+dist_noinst_DATA += %D%/openzfs-zfs-modules-_KVERS_-di.install.in
+dist_noinst_DATA += %D%/openzfs-zfs-modules-_KVERS_.install.in
+dist_noinst_DATA += %D%/openzfs-zfs-modules-_KVERS_.postinst.in
+dist_noinst_DATA += %D%/openzfs-zfs-modules-_KVERS_.postrm.in
+dist_noinst_DATA += %D%/openzfs-zfs-test.install
+dist_noinst_DATA += %D%/openzfs-zfsutils.docs
+dist_noinst_DATA += %D%/openzfs-zfsutils.examples
+dist_noinst_DATA += %D%/openzfs-zfsutils.install
+dist_noinst_DATA += %D%/openzfs-zfsutils.postinst
+dist_noinst_DATA += %D%/openzfs-zfs-zed.install
+dist_noinst_DATA += %D%/openzfs-zfs-zed.postinst
+dist_noinst_DATA += %D%/openzfs-zfs-zed.postrm
+dist_noinst_DATA += %D%/rules.in
+dist_noinst_DATA += %D%/source
+dist_noinst_DATA += %D%/tree

contrib/debian/rules.in

@@ -56,6 +56,12 @@ override_dh_auto_configure:
 		sed 's/@DEB_HOST_MULTIARCH@/$(DEB_HOST_MULTIARCH)/g' "$$i" > "$${i%%.in}" ; \
 	done
 
+	ln -s '$(CURDIR)/etc/init.d/zfs-import' '$(CURDIR)/debian/openzfs-zfsutils.zfs-import.init'
+	ln -s '$(CURDIR)/etc/init.d/zfs-load-key' '$(CURDIR)/debian/openzfs-zfsutils.zfs-load-key.init'
+	ln -s '$(CURDIR)/etc/init.d/zfs-mount' '$(CURDIR)/debian/openzfs-zfsutils.zfs-mount.init'
+	ln -s '$(CURDIR)/etc/init.d/zfs-share' '$(CURDIR)/debian/openzfs-zfsutils.zfs-share.init'
+	ln -s '$(CURDIR)/etc/init.d/zfs-zed' '$(CURDIR)/debian/openzfs-zfs-zed.zfs-zed.init'
+
 override_dh_gencontrol:
 	dh_gencontrol -- -Vlinux:Recommends="linux-libc-dev (<< $(LINUX_NEXT)~), linux-libc-dev (>= $(LINUX_MIN)~),"
 
@@ -82,6 +88,7 @@ override_dh_auto_install:
 
 	@# Zed has dependencies outside of the system root.
 	mv '$(CURDIR)/debian/tmp/sbin/zed' '$(CURDIR)/debian/tmp/usr/sbin/zed'
+	sed -i 's|ExecStart=/sbin/|ExecStart=/usr/sbin/|g' '$(CURDIR)/debian/tmp/lib/systemd/system/zfs-zed.service'
 
 	@# Install the DKMS source.
 	@# We only want the files needed to build the modules

include/sys/zap.h

@@ -377,6 +377,7 @@ typedef struct {
 	uint64_t za_num_integers;
 	uint64_t za_first_integer;	/* no sign extension for <8byte ints */
 	uint32_t za_name_len;
+	uint32_t za_pad;	/* We want za_name aligned to uint64_t. */
 	char za_name[];
 } zap_attribute_t;
 

man/man4/zfs.4

@@ -2012,6 +2012,13 @@ Ignore the
 feature, causing an operation that would start a resilver to
 immediately restart the one in progress.
 .
+.It Sy zfs_resilver_defer_percent Ns = Ns Sy 10 Ns % Pq uint
+If the ongoing resilver progress is below this threshold, a new resilver will
+restart from scratch instead of being deferred after the current one finishes,
+even if the
+.Sy resilver_defer
+feature is enabled.
+.
 .It Sy zfs_resilver_min_time_ms Ns = Ns Sy 3000 Ns ms Po 3 s Pc Pq uint
 Resilvers are processed by the sync thread.
 While resilvering, it will spend at least this much time

module/zfs/arc.c

@@ -4251,7 +4251,7 @@ arc_mf(uint64_t x, uint64_t multiplier, uint64_t divisor)
 static uint64_t
 arc_evict(void)
 {
-	uint64_t asize, bytes, total_evicted = 0;
+	uint64_t bytes, total_evicted = 0;
 	int64_t e, mrud, mrum, mfud, mfum, w;
 	static uint64_t ogrd, ogrm, ogfd, ogfm;
 	static uint64_t gsrd, gsrm, gsfd, gsfm;
@@ -4288,8 +4288,9 @@ arc_evict(void)
 	arc_pd = arc_evict_adj(arc_pd, gsrd + gsfd, grd, gfd, 100);
 	arc_pm = arc_evict_adj(arc_pm, gsrm + gsfm, grm, gfm, 100);
 
-	asize = aggsum_value(&arc_sums.arcstat_size);
-	int64_t wt = t - (asize - arc_c);
+	uint64_t asize = aggsum_value(&arc_sums.arcstat_size);
+	uint64_t ac = arc_c;
+	int64_t wt = t - (asize - ac);
 
 	/*
 	 * Try to reduce pinned dnodes if more than 3/4 of wanted metadata
@@ -4317,15 +4318,15 @@ arc_evict(void)
 
 	/* Evict MRU metadata. */
 	w = wt * (int64_t)(arc_meta * arc_pm >> 48) >> 16;
-	e = MIN((int64_t)(asize - arc_c), (int64_t)(mrum - w));
+	e = MIN((int64_t)(asize - ac), (int64_t)(mrum - w));
 	bytes = arc_evict_impl(arc_mru, ARC_BUFC_METADATA, e);
 	total_evicted += bytes;
 	mrum -= bytes;
 	asize -= bytes;
 
 	/* Evict MFU metadata. */
 	w = wt * (int64_t)(arc_meta >> 16) >> 16;
-	e = MIN((int64_t)(asize - arc_c), (int64_t)(m - bytes - w));
+	e = MIN((int64_t)(asize - ac), (int64_t)(m - bytes - w));
 	bytes = arc_evict_impl(arc_mfu, ARC_BUFC_METADATA, e);
 	total_evicted += bytes;
 	mfum -= bytes;
@@ -4334,14 +4335,14 @@ arc_evict(void)
 	/* Evict MRU data. */
 	wt -= m - total_evicted;
 	w = wt * (int64_t)(arc_pd >> 16) >> 16;
-	e = MIN((int64_t)(asize - arc_c), (int64_t)(mrud - w));
+	e = MIN((int64_t)(asize - ac), (int64_t)(mrud - w));
 	bytes = arc_evict_impl(arc_mru, ARC_BUFC_DATA, e);
 	total_evicted += bytes;
 	mrud -= bytes;
 	asize -= bytes;
 
 	/* Evict MFU data. */
-	e = asize - arc_c;
+	e = asize - ac;
 	bytes = arc_evict_impl(arc_mfu, ARC_BUFC_DATA, e);
 	mfud -= bytes;
 	total_evicted += bytes;

module/zfs/dsl_scan.c

@@ -212,6 +212,9 @@ static uint64_t zfs_max_async_dedup_frees = 100000;
 /* set to disable resilver deferring */
 static int zfs_resilver_disable_defer = B_FALSE;
 
+/* Don't defer a resilver if the one in progress only got this far: */
+static uint_t zfs_resilver_defer_percent = 10;
+
 /*
  * We wait a few txgs after importing a pool to begin scanning so that
  * the import / mounting code isn't held up by scrub / resilver IO.
@@ -4289,35 +4292,54 @@ dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx)
 	dsl_scan_t *scn = dp->dp_scan;
 	spa_t *spa = dp->dp_spa;
 	state_sync_type_t sync_type = SYNC_OPTIONAL;
+	int restart_early = 0;
+
+	if (spa->spa_resilver_deferred) {
+		uint64_t to_issue, issued;
+
+		if (!spa_feature_is_active(dp->dp_spa,
+		    SPA_FEATURE_RESILVER_DEFER))
+			spa_feature_incr(spa, SPA_FEATURE_RESILVER_DEFER, tx);
+
+		/*
+		 * See print_scan_scrub_resilver_status() issued/total_i
+		 * @ cmd/zpool/zpool_main.c
+		 */
+		to_issue =
+		    scn->scn_phys.scn_to_examine - scn->scn_phys.scn_skipped;
+		issued =
+		    scn->scn_issued_before_pass + spa->spa_scan_pass_issued;
+		restart_early =
+		    zfs_resilver_disable_defer ||
+		    (issued < (to_issue * zfs_resilver_defer_percent / 100));
+	}
+
+	/*
+	 * Only process scans in sync pass 1.
+	 */
+	if (spa_sync_pass(spa) > 1)
+		return;
 
-	if (spa->spa_resilver_deferred &&
-	    !spa_feature_is_active(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER))
-		spa_feature_incr(spa, SPA_FEATURE_RESILVER_DEFER, tx);
 
 	/*
 	 * Check for scn_restart_txg before checking spa_load_state, so
 	 * that we can restart an old-style scan while the pool is being
 	 * imported (see dsl_scan_init). We also restart scans if there
 	 * is a deferred resilver and the user has manually disabled
-	 * deferred resilvers via the tunable.
+	 * deferred resilvers via zfs_resilver_disable_defer, or if the
+	 * current scan progress is below zfs_resilver_defer_percent.
 	 */
-	if (dsl_scan_restarting(scn, tx) ||
-	    (spa->spa_resilver_deferred && zfs_resilver_disable_defer)) {
+	if (dsl_scan_restarting(scn, tx) || restart_early) {
 		pool_scan_func_t func = POOL_SCAN_SCRUB;
 		dsl_scan_done(scn, B_FALSE, tx);
 		if (vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL))
 			func = POOL_SCAN_RESILVER;
-		zfs_dbgmsg("restarting scan func=%u on %s txg=%llu",
-		    func, dp->dp_spa->spa_name, (longlong_t)tx->tx_txg);
+		zfs_dbgmsg("restarting scan func=%u on %s txg=%llu early=%d",
+		    func, dp->dp_spa->spa_name, (longlong_t)tx->tx_txg,
+		    restart_early);
 		dsl_scan_setup_sync(&func, tx);
 	}
 
-	/*
-	 * Only process scans in sync pass 1.
-	 */
-	if (spa_sync_pass(spa) > 1)
-		return;
-
 	/*
 	 * If the spa is shutting down, then stop scanning. This will
 	 * ensure that the scan does not dirty any new data during the
@@ -5287,6 +5309,9 @@ ZFS_MODULE_PARAM(zfs, zfs_, scan_report_txgs, UINT, ZMOD_RW,
 ZFS_MODULE_PARAM(zfs, zfs_, resilver_disable_defer, INT, ZMOD_RW,
 	"Process all resilvers immediately");
 
+ZFS_MODULE_PARAM(zfs, zfs_, resilver_defer_percent, UINT, ZMOD_RW,
+	"Issued IO percent complete after which resilvers are deferred");
+
 ZFS_MODULE_PARAM(zfs, zfs_, scrub_error_blocks_per_txg, UINT, ZMOD_RW,
 	"Error blocks to be scrubbed in one txg");
 /* END CSTYLED */

tests/zfs-tests/include/tunables.cfg

@@ -73,6 +73,7 @@ REBUILD_SCRUB_ENABLED		rebuild_scrub_enabled		zfs_rebuild_scrub_enabled
 REMOVAL_SUSPEND_PROGRESS	removal_suspend_progress	zfs_removal_suspend_progress
 REMOVE_MAX_SEGMENT		remove_max_segment		zfs_remove_max_segment
 RESILVER_MIN_TIME_MS		resilver_min_time_ms		zfs_resilver_min_time_ms
+RESILVER_DEFER_PERCENT		resilver_defer_percent		zfs_resilver_defer_percent
 SCAN_LEGACY			scan_legacy			zfs_scan_legacy
 SCAN_SUSPEND_PROGRESS		scan_suspend_progress		zfs_scan_suspend_progress
 SCAN_VDEV_LIMIT			scan_vdev_limit			zfs_scan_vdev_limit

tests/zfs-tests/tests/functional/replacement/resilver_restart_001.ksh

@@ -96,6 +96,8 @@ set -A RESTARTS -- '1' '2' '2' '2'
 set -A VDEVS -- '' '' '' ''
 set -A DEFER_RESTARTS -- '1' '1' '1' '2'
 set -A DEFER_VDEVS -- '-' '2' '2' '-'
+set -A EARLY_RESTART_DEFER_RESTARTS -- '1' '2' '2' '2'
+set -A EARLY_RESTART_DEFER_VDEVS -- '' '' '' ''
 
 VDEV_REPLACE="${VDEV_FILES[1]} $SPARE_VDEV_FILE"
 
@@ -125,7 +127,7 @@ done
 wait
 
 # test without and with deferred resilve feature enabled
-for test in "without" "with"
+for test in "without" "with" "with_early_restart"
 do
 	log_note "Testing $test deferred resilvers"
 
@@ -135,6 +137,13 @@ do
 		RESTARTS=( "${DEFER_RESTARTS[@]}" )
 		VDEVS=( "${DEFER_VDEVS[@]}" )
 		VDEV_REPLACE="$SPARE_VDEV_FILE ${VDEV_FILES[1]}"
+		log_must set_tunable32 RESILVER_DEFER_PERCENT 0
+	elif [[ $test == "with_early_restart" ]]
+	then
+		RESTARTS=( "${EARLY_RESTART_DEFER_RESTARTS[@]}" )
+		VDEVS=( "${EARLY_RESTART_DEFER_VDEVS[@]}" )
+		VDEV_REPLACE="${VDEV_FILES[1]} $SPARE_VDEV_FILE"
+		log_must set_tunable32 RESILVER_DEFER_PERCENT 100
 	fi
 
 	# clear the events