xref: /freebsd-13-stable/sys/contrib/openzfs/tests/zfs-tests/include/libtest.shlib (revision b9c2c366db1beb2ed276947056f45938ad8f57ec)
1#
2# CDDL HEADER START
3#
4# The contents of this file are subject to the terms of the
5# Common Development and Distribution License (the "License").
6# You may not use this file except in compliance with the License.
7#
8# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9# or http://www.opensolaris.org/os/licensing.
10# See the License for the specific language governing permissions
11# and limitations under the License.
12#
13# When distributing Covered Code, include this CDDL HEADER in each
14# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15# If applicable, add the following below this CDDL HEADER, with the
16# fields enclosed by brackets "[]" replaced with your own identifying
17# information: Portions Copyright [yyyy] [name of copyright owner]
18#
19# CDDL HEADER END
20#
21
22#
23# Copyright (c) 2009, Sun Microsystems Inc. All rights reserved.
24# Copyright (c) 2012, 2020, Delphix. All rights reserved.
25# Copyright (c) 2017, Tim Chase. All rights reserved.
26# Copyright (c) 2017, Nexenta Systems Inc. All rights reserved.
27# Copyright (c) 2017, Lawrence Livermore National Security LLC.
28# Copyright (c) 2017, Datto Inc. All rights reserved.
29# Copyright (c) 2017, Open-E Inc. All rights reserved.
30# Copyright (c) 2021, The FreeBSD Foundation.
31# Use is subject to license terms.
32#
33
34. ${STF_TOOLS}/include/logapi.shlib
35. ${STF_SUITE}/include/math.shlib
36. ${STF_SUITE}/include/blkdev.shlib
37
38. ${STF_SUITE}/include/tunables.cfg
39
40# On AlmaLinux 9 we will see $PWD = '.' instead of the full path.  This causes
41# some tests to fail.  Fix it up here.
42if [ "$PWD" = "." ] ; then
43	PWD="$(readlink -f $PWD)"
44fi
45
46#
47# Apply constrained path when available.  This is required since the
48# PATH may have been modified by sudo's secure_path behavior.
49#
50if [ -n "$STF_PATH" ]; then
51	export PATH="$STF_PATH"
52fi
53
54#
55# Generic dot version comparison function
56#
57# Returns success when version $1 is greater than or equal to $2.
58#
59function compare_version_gte
60{
61	if [[ "$(printf "$1\n$2" | sort -V | tail -n1)" == "$1" ]]; then
62		return 0
63	else
64		return 1
65	fi
66}
67
68# Linux kernel version comparison function
69#
70# $1 Linux version ("4.10", "2.6.32") or blank for installed Linux version
71#
72# Used for comparison: if [ $(linux_version) -ge $(linux_version "2.6.32") ]
73#
74function linux_version
75{
76	typeset ver="$1"
77
78	[[ -z "$ver" ]] && ver=$(uname -r | grep -Eo "^[0-9]+\.[0-9]+\.[0-9]+")
79
80	typeset version=$(echo $ver | cut -d '.' -f 1)
81	typeset major=$(echo $ver | cut -d '.' -f 2)
82	typeset minor=$(echo $ver | cut -d '.' -f 3)
83
84	[[ -z "$version" ]] && version=0
85	[[ -z "$major" ]] && major=0
86	[[ -z "$minor" ]] && minor=0
87
88	echo $((version * 10000 + major * 100 + minor))
89}
90
91# Determine if this is a Linux test system
92#
93# Return 0 if platform Linux, 1 if otherwise
94
95function is_linux
96{
97	if [[ $(uname -o) == "GNU/Linux" ]]; then
98		return 0
99	else
100		return 1
101	fi
102}
103
104# Determine if this is an illumos test system
105#
106# Return 0 if platform illumos, 1 if otherwise
107function is_illumos
108{
109	if [[ $(uname -o) == "illumos" ]]; then
110		return 0
111	else
112		return 1
113	fi
114}
115
116# Determine if this is a FreeBSD test system
117#
118# Return 0 if platform FreeBSD, 1 if otherwise
119
120function is_freebsd
121{
122	if [[ $(uname -o) == "FreeBSD" ]]; then
123		return 0
124	else
125		return 1
126	fi
127}
128
129# Determine if this is a DilOS test system
130#
131# Return 0 if platform DilOS, 1 if otherwise
132
133function is_dilos
134{
135	typeset ID=""
136	[[ -f /etc/os-release ]] && . /etc/os-release
137	if [[ $ID == "dilos" ]]; then
138		return 0
139	else
140		return 1
141	fi
142}
143
144# Determine if this is a 32-bit system
145#
146# Return 0 if platform is 32-bit, 1 if otherwise
147
148function is_32bit
149{
150	if [[ $(getconf LONG_BIT) == "32" ]]; then
151		return 0
152	else
153		return 1
154	fi
155}
156
157# Determine if kmemleak is enabled
158#
159# Return 0 if kmemleak is enabled, 1 if otherwise
160
161function is_kmemleak
162{
163	if is_linux && [[ -e /sys/kernel/debug/kmemleak ]]; then
164		return 0
165	else
166		return 1
167	fi
168}
169
170# Determine whether a dataset is mounted
171#
172# $1 dataset name
173# $2 filesystem type; optional - defaulted to zfs
174#
175# Return 0 if dataset is mounted; 1 if unmounted; 2 on error
176
177function ismounted
178{
179	typeset fstype=$2
180	[[ -z $fstype ]] && fstype=zfs
181	typeset out dir name ret
182
183	case $fstype in
184		zfs)
185			if [[ "$1" == "/"* ]] ; then
186				for out in $(zfs mount | awk '{print $2}'); do
187					[[ $1 == $out ]] && return 0
188				done
189			else
190				for out in $(zfs mount | awk '{print $1}'); do
191					[[ $1 == $out ]] && return 0
192				done
193			fi
194		;;
195		ufs|nfs)
196			if is_freebsd; then
197				mount -pt $fstype | while read dev dir _t _flags; do
198					[[ "$1" == "$dev" || "$1" == "$dir" ]] && return 0
199				done
200			else
201				out=$(df -F $fstype $1 2>/dev/null)
202				ret=$?
203				(($ret != 0)) && return $ret
204
205				dir=${out%%\(*}
206				dir=${dir%% *}
207				name=${out##*\(}
208				name=${name%%\)*}
209				name=${name%% *}
210
211				[[ "$1" == "$dir" || "$1" == "$name" ]] && return 0
212			fi
213		;;
214		ext*)
215			out=$(df -t $fstype $1 2>/dev/null)
216			return $?
217		;;
218		zvol)
219			if [[ -L "$ZVOL_DEVDIR/$1" ]]; then
220				link=$(readlink -f $ZVOL_DEVDIR/$1)
221				[[ -n "$link" ]] && \
222					mount | grep -q "^$link" && \
223						return 0
224			fi
225		;;
226	esac
227
228	return 1
229}
230
231# Return 0 if a dataset is mounted; 1 otherwise
232#
233# $1 dataset name
234# $2 filesystem type; optional - defaulted to zfs
235
236function mounted
237{
238	ismounted $1 $2
239	(($? == 0)) && return 0
240	return 1
241}
242
243# Return 0 if a dataset is unmounted; 1 otherwise
244#
245# $1 dataset name
246# $2 filesystem type; optional - defaulted to zfs
247
248function unmounted
249{
250	ismounted $1 $2
251	(($? == 1)) && return 0
252	return 1
253}
254
255# split line on ","
256#
257# $1 - line to split
258
259function splitline
260{
261	echo $1 | tr ',' ' '
262}
263
264function default_setup
265{
266	default_setup_noexit "$@"
267
268	log_pass
269}
270
271function default_setup_no_mountpoint
272{
273	default_setup_noexit "$1" "$2" "$3" "yes"
274
275	log_pass
276}
277
278#
279# Given a list of disks, setup storage pools and datasets.
280#
281function default_setup_noexit
282{
283	typeset disklist=$1
284	typeset container=$2
285	typeset volume=$3
286	typeset no_mountpoint=$4
287	log_note begin default_setup_noexit
288
289	if is_global_zone; then
290		if poolexists $TESTPOOL ; then
291			destroy_pool $TESTPOOL
292		fi
293		[[ -d /$TESTPOOL ]] && rm -rf /$TESTPOOL
294		log_must zpool create -f $TESTPOOL $disklist
295	else
296		reexport_pool
297	fi
298
299	rm -rf $TESTDIR  || log_unresolved Could not remove $TESTDIR
300	mkdir -p $TESTDIR || log_unresolved Could not create $TESTDIR
301
302	log_must zfs create $TESTPOOL/$TESTFS
303	if [[ -z $no_mountpoint ]]; then
304		log_must zfs set mountpoint=$TESTDIR $TESTPOOL/$TESTFS
305	fi
306
307	if [[ -n $container ]]; then
308		rm -rf $TESTDIR1  || \
309			log_unresolved Could not remove $TESTDIR1
310		mkdir -p $TESTDIR1 || \
311			log_unresolved Could not create $TESTDIR1
312
313		log_must zfs create $TESTPOOL/$TESTCTR
314		log_must zfs set canmount=off $TESTPOOL/$TESTCTR
315		log_must zfs create $TESTPOOL/$TESTCTR/$TESTFS1
316		if [[ -z $no_mountpoint ]]; then
317			log_must zfs set mountpoint=$TESTDIR1 \
318			    $TESTPOOL/$TESTCTR/$TESTFS1
319		fi
320	fi
321
322	if [[ -n $volume ]]; then
323		if is_global_zone ; then
324			log_must zfs create -V $VOLSIZE $TESTPOOL/$TESTVOL
325			block_device_wait
326		else
327			log_must zfs create $TESTPOOL/$TESTVOL
328		fi
329	fi
330}
331
332#
333# Given a list of disks, setup a storage pool, file system and
334# a container.
335#
336function default_container_setup
337{
338	typeset disklist=$1
339
340	default_setup "$disklist" "true"
341}
342
343#
344# Given a list of disks, setup a storage pool,file system
345# and a volume.
346#
347function default_volume_setup
348{
349	typeset disklist=$1
350
351	default_setup "$disklist" "" "true"
352}
353
354#
355# Given a list of disks, setup a storage pool,file system,
356# a container and a volume.
357#
358function default_container_volume_setup
359{
360	typeset disklist=$1
361
362	default_setup "$disklist" "true" "true"
363}
364
365#
366# Create a snapshot on a filesystem or volume. Defaultly create a snapshot on
367# filesystem
368#
369# $1 Existing filesystem or volume name. Default, $TESTPOOL/$TESTFS
370# $2 snapshot name. Default, $TESTSNAP
371#
372function create_snapshot
373{
374	typeset fs_vol=${1:-$TESTPOOL/$TESTFS}
375	typeset snap=${2:-$TESTSNAP}
376
377	[[ -z $fs_vol ]] && log_fail "Filesystem or volume's name is undefined."
378	[[ -z $snap ]] && log_fail "Snapshot's name is undefined."
379
380	if snapexists $fs_vol@$snap; then
381		log_fail "$fs_vol@$snap already exists."
382	fi
383	datasetexists $fs_vol || \
384		log_fail "$fs_vol must exist."
385
386	log_must zfs snapshot $fs_vol@$snap
387}
388
389#
390# Create a clone from a snapshot, default clone name is $TESTCLONE.
391#
392# $1 Existing snapshot, $TESTPOOL/$TESTFS@$TESTSNAP is default.
393# $2 Clone name, $TESTPOOL/$TESTCLONE is default.
394#
395function create_clone   # snapshot clone
396{
397	typeset snap=${1:-$TESTPOOL/$TESTFS@$TESTSNAP}
398	typeset clone=${2:-$TESTPOOL/$TESTCLONE}
399
400	[[ -z $snap ]] && \
401		log_fail "Snapshot name is undefined."
402	[[ -z $clone ]] && \
403		log_fail "Clone name is undefined."
404
405	log_must zfs clone $snap $clone
406}
407
408#
409# Create a bookmark of the given snapshot.  Defaultly create a bookmark on
410# filesystem.
411#
412# $1 Existing filesystem or volume name. Default, $TESTFS
413# $2 Existing snapshot name. Default, $TESTSNAP
414# $3 bookmark name. Default, $TESTBKMARK
415#
416function create_bookmark
417{
418	typeset fs_vol=${1:-$TESTFS}
419	typeset snap=${2:-$TESTSNAP}
420	typeset bkmark=${3:-$TESTBKMARK}
421
422	[[ -z $fs_vol ]] && log_fail "Filesystem or volume's name is undefined."
423	[[ -z $snap ]] && log_fail "Snapshot's name is undefined."
424	[[ -z $bkmark ]] && log_fail "Bookmark's name is undefined."
425
426	if bkmarkexists $fs_vol#$bkmark; then
427		log_fail "$fs_vol#$bkmark already exists."
428	fi
429	datasetexists $fs_vol || \
430		log_fail "$fs_vol must exist."
431	snapexists $fs_vol@$snap || \
432		log_fail "$fs_vol@$snap must exist."
433
434	log_must zfs bookmark $fs_vol@$snap $fs_vol#$bkmark
435}
436
437#
438# Create a temporary clone result of an interrupted resumable 'zfs receive'
439# $1 Destination filesystem name. Must not exist, will be created as the result
440#    of this function along with its %recv temporary clone
441# $2 Source filesystem name. Must not exist, will be created and destroyed
442#
443function create_recv_clone
444{
445	typeset recvfs="$1"
446	typeset sendfs="${2:-$TESTPOOL/create_recv_clone}"
447	typeset snap="$sendfs@snap1"
448	typeset incr="$sendfs@snap2"
449	typeset mountpoint="$TESTDIR/create_recv_clone"
450	typeset sendfile="$TESTDIR/create_recv_clone.zsnap"
451
452	[[ -z $recvfs ]] && log_fail "Recv filesystem's name is undefined."
453
454	datasetexists $recvfs && log_fail "Recv filesystem must not exist."
455	datasetexists $sendfs && log_fail "Send filesystem must not exist."
456
457	log_must zfs create -o mountpoint="$mountpoint" $sendfs
458	log_must zfs snapshot $snap
459	log_must eval "zfs send $snap | zfs recv -u $recvfs"
460	log_must mkfile 1m "$mountpoint/data"
461	log_must zfs snapshot $incr
462	log_must eval "zfs send -i $snap $incr | dd bs=10K count=1 \
463	    iflag=fullblock > $sendfile"
464	log_mustnot eval "zfs recv -su $recvfs < $sendfile"
465	destroy_dataset "$sendfs" "-r"
466	log_must rm -f "$sendfile"
467
468	if [[ $(get_prop 'inconsistent' "$recvfs/%recv") -ne 1 ]]; then
469		log_fail "Error creating temporary $recvfs/%recv clone"
470	fi
471}
472
473function default_mirror_setup
474{
475	default_mirror_setup_noexit $1 $2 $3
476
477	log_pass
478}
479
480#
481# Given a pair of disks, set up a storage pool and dataset for the mirror
482# @parameters: $1 the primary side of the mirror
483#   $2 the secondary side of the mirror
484# @uses: ZPOOL ZFS TESTPOOL TESTFS
485function default_mirror_setup_noexit
486{
487	readonly func="default_mirror_setup_noexit"
488	typeset primary=$1
489	typeset secondary=$2
490
491	[[ -z $primary ]] && \
492		log_fail "$func: No parameters passed"
493	[[ -z $secondary ]] && \
494		log_fail "$func: No secondary partition passed"
495	[[ -d /$TESTPOOL ]] && rm -rf /$TESTPOOL
496	log_must zpool create -f $TESTPOOL mirror $@
497	log_must zfs create $TESTPOOL/$TESTFS
498	log_must zfs set mountpoint=$TESTDIR $TESTPOOL/$TESTFS
499}
500
501#
502# create a number of mirrors.
503# We create a number($1) of 2 way mirrors using the pairs of disks named
504# on the command line. These mirrors are *not* mounted
505# @parameters: $1 the number of mirrors to create
506#  $... the devices to use to create the mirrors on
507# @uses: ZPOOL ZFS TESTPOOL
508function setup_mirrors
509{
510	typeset -i nmirrors=$1
511
512	shift
513	while ((nmirrors > 0)); do
514		log_must test -n "$1" -a -n "$2"
515		[[ -d /$TESTPOOL$nmirrors ]] && rm -rf /$TESTPOOL$nmirrors
516		log_must zpool create -f $TESTPOOL$nmirrors mirror $1 $2
517		shift 2
518		((nmirrors = nmirrors - 1))
519	done
520}
521
522#
523# create a number of raidz pools.
524# We create a number($1) of 2 raidz pools  using the pairs of disks named
525# on the command line. These pools are *not* mounted
526# @parameters: $1 the number of pools to create
527#  $... the devices to use to create the pools on
528# @uses: ZPOOL ZFS TESTPOOL
529function setup_raidzs
530{
531	typeset -i nraidzs=$1
532
533	shift
534	while ((nraidzs > 0)); do
535		log_must test -n "$1" -a -n "$2"
536		[[ -d /$TESTPOOL$nraidzs ]] && rm -rf /$TESTPOOL$nraidzs
537		log_must zpool create -f $TESTPOOL$nraidzs raidz $1 $2
538		shift 2
539		((nraidzs = nraidzs - 1))
540	done
541}
542
543#
544# Destroy the configured testpool mirrors.
545# the mirrors are of the form ${TESTPOOL}{number}
546# @uses: ZPOOL ZFS TESTPOOL
547function destroy_mirrors
548{
549	default_cleanup_noexit
550
551	log_pass
552}
553
554#
555# Given a minimum of two disks, set up a storage pool and dataset for the raid-z
556# $1 the list of disks
557#
558function default_raidz_setup
559{
560	typeset disklist="$*"
561	disks=(${disklist[*]})
562
563	if [[ ${#disks[*]} -lt 2 ]]; then
564		log_fail "A raid-z requires a minimum of two disks."
565	fi
566
567	[[ -d /$TESTPOOL ]] && rm -rf /$TESTPOOL
568	log_must zpool create -f $TESTPOOL raidz $disklist
569	log_must zfs create $TESTPOOL/$TESTFS
570	log_must zfs set mountpoint=$TESTDIR $TESTPOOL/$TESTFS
571
572	log_pass
573}
574
575#
576# Common function used to cleanup storage pools and datasets.
577#
578# Invoked at the start of the test suite to ensure the system
579# is in a known state, and also at the end of each set of
580# sub-tests to ensure errors from one set of tests doesn't
581# impact the execution of the next set.
582
583function default_cleanup
584{
585	default_cleanup_noexit
586
587	log_pass
588}
589
590#
591# Utility function used to list all available pool names.
592#
593# NOTE: $KEEP is a variable containing pool names, separated by a newline
594# character, that must be excluded from the returned list.
595#
596function get_all_pools
597{
598	zpool list -H -o name | grep -Fvx "$KEEP" | grep -v "$NO_POOLS"
599}
600
601function default_cleanup_noexit
602{
603	typeset pool=""
604	#
605	# Destroying the pool will also destroy any
606	# filesystems it contains.
607	#
608	if is_global_zone; then
609		zfs unmount -a > /dev/null 2>&1
610		ALL_POOLS=$(get_all_pools)
611		# Here, we loop through the pools we're allowed to
612		# destroy, only destroying them if it's safe to do
613		# so.
614		while [ ! -z ${ALL_POOLS} ]
615		do
616			for pool in ${ALL_POOLS}
617			do
618				if safe_to_destroy_pool $pool ;
619				then
620					destroy_pool $pool
621				fi
622			done
623			ALL_POOLS=$(get_all_pools)
624		done
625
626		zfs mount -a
627	else
628		typeset fs=""
629		for fs in $(zfs list -H -o name \
630		    | grep "^$ZONE_POOL/$ZONE_CTR[01234]/"); do
631			destroy_dataset "$fs" "-Rf"
632		done
633
634		# Need cleanup here to avoid garbage dir left.
635		for fs in $(zfs list -H -o name); do
636			[[ $fs == /$ZONE_POOL ]] && continue
637			[[ -d $fs ]] && log_must rm -rf $fs/*
638		done
639
640		#
641		# Reset the $ZONE_POOL/$ZONE_CTR[01234] file systems property to
642		# the default value
643		#
644		for fs in $(zfs list -H -o name); do
645			if [[ $fs == $ZONE_POOL/$ZONE_CTR[01234] ]]; then
646				log_must zfs set reservation=none $fs
647				log_must zfs set recordsize=128K $fs
648				log_must zfs set mountpoint=/$fs $fs
649				typeset enc=""
650				enc=$(get_prop encryption $fs)
651				if [[ $? -ne 0 ]] || [[ -z "$enc" ]] || \
652					[[ "$enc" == "off" ]]; then
653					log_must zfs set checksum=on $fs
654				fi
655				log_must zfs set compression=off $fs
656				log_must zfs set atime=on $fs
657				log_must zfs set devices=off $fs
658				log_must zfs set exec=on $fs
659				log_must zfs set setuid=on $fs
660				log_must zfs set readonly=off $fs
661				log_must zfs set snapdir=hidden $fs
662				log_must zfs set aclmode=groupmask $fs
663				log_must zfs set aclinherit=secure $fs
664			fi
665		done
666	fi
667
668	[[ -d $TESTDIR ]] && \
669		log_must rm -rf $TESTDIR
670
671	disk1=${DISKS%% *}
672	if is_mpath_device $disk1; then
673		delete_partitions
674	fi
675
676	rm -f $TEST_BASE_DIR/{err,out}
677}
678
679
680#
681# Common function used to cleanup storage pools, file systems
682# and containers.
683#
684function default_container_cleanup
685{
686	if ! is_global_zone; then
687		reexport_pool
688	fi
689
690	ismounted $TESTPOOL/$TESTCTR/$TESTFS1
691	[[ $? -eq 0 ]] && \
692	    log_must zfs unmount $TESTPOOL/$TESTCTR/$TESTFS1
693
694	destroy_dataset "$TESTPOOL/$TESTCTR/$TESTFS1" "-R"
695	destroy_dataset "$TESTPOOL/$TESTCTR" "-Rf"
696
697	[[ -e $TESTDIR1 ]] && \
698	    log_must rm -rf $TESTDIR1 > /dev/null 2>&1
699
700	default_cleanup
701}
702
703#
704# Common function used to cleanup snapshot of file system or volume. Default to
705# delete the file system's snapshot
706#
707# $1 snapshot name
708#
709function destroy_snapshot
710{
711	typeset snap=${1:-$TESTPOOL/$TESTFS@$TESTSNAP}
712
713	if ! snapexists $snap; then
714		log_fail "'$snap' does not exist."
715	fi
716
717	#
718	# For the sake of the value which come from 'get_prop' is not equal
719	# to the really mountpoint when the snapshot is unmounted. So, firstly
720	# check and make sure this snapshot's been mounted in current system.
721	#
722	typeset mtpt=""
723	if ismounted $snap; then
724		mtpt=$(get_prop mountpoint $snap)
725		(($? != 0)) && \
726			log_fail "get_prop mountpoint $snap failed."
727	fi
728
729	destroy_dataset "$snap"
730	[[ $mtpt != "" && -d $mtpt ]] && \
731		log_must rm -rf $mtpt
732}
733
734#
735# Common function used to cleanup clone.
736#
737# $1 clone name
738#
739function destroy_clone
740{
741	typeset clone=${1:-$TESTPOOL/$TESTCLONE}
742
743	if ! datasetexists $clone; then
744		log_fail "'$clone' does not existed."
745	fi
746
747	# With the same reason in destroy_snapshot
748	typeset mtpt=""
749	if ismounted $clone; then
750		mtpt=$(get_prop mountpoint $clone)
751		(($? != 0)) && \
752			log_fail "get_prop mountpoint $clone failed."
753	fi
754
755	destroy_dataset "$clone"
756	[[ $mtpt != "" && -d $mtpt ]] && \
757		log_must rm -rf $mtpt
758}
759
760#
761# Common function used to cleanup bookmark of file system or volume.  Default
762# to delete the file system's bookmark.
763#
764# $1 bookmark name
765#
766function destroy_bookmark
767{
768	typeset bkmark=${1:-$TESTPOOL/$TESTFS#$TESTBKMARK}
769
770	if ! bkmarkexists $bkmark; then
771		log_fail "'$bkmarkp' does not existed."
772	fi
773
774	destroy_dataset "$bkmark"
775}
776
777# Return 0 if a snapshot exists; $? otherwise
778#
779# $1 - snapshot name
780
781function snapexists
782{
783	zfs list -H -t snapshot "$1" > /dev/null 2>&1
784	return $?
785}
786
787#
788# Return 0 if a bookmark exists; $? otherwise
789#
790# $1 - bookmark name
791#
792function bkmarkexists
793{
794	zfs list -H -t bookmark "$1" > /dev/null 2>&1
795	return $?
796}
797
798#
799# Return 0 if a hold exists; $? otherwise
800#
801# $1 - hold tag
802# $2 - snapshot name
803#
804function holdexists
805{
806	zfs holds "$2" | awk '{ print $2 }' | grep "$1" > /dev/null 2>&1
807	return $?
808}
809
810#
811# Set a property to a certain value on a dataset.
812# Sets a property of the dataset to the value as passed in.
813# @param:
814#	$1 dataset who's property is being set
815#	$2 property to set
816#	$3 value to set property to
817# @return:
818#	0 if the property could be set.
819#	non-zero otherwise.
820# @use: ZFS
821#
822function dataset_setprop
823{
824	typeset fn=dataset_setprop
825
826	if (($# < 3)); then
827		log_note "$fn: Insufficient parameters (need 3, had $#)"
828		return 1
829	fi
830	typeset output=
831	output=$(zfs set $2=$3 $1 2>&1)
832	typeset rv=$?
833	if ((rv != 0)); then
834		log_note "Setting property on $1 failed."
835		log_note "property $2=$3"
836		log_note "Return Code: $rv"
837		log_note "Output: $output"
838		return $rv
839	fi
840	return 0
841}
842
843#
844# Assign suite defined dataset properties.
845# This function is used to apply the suite's defined default set of
846# properties to a dataset.
847# @parameters: $1 dataset to use
848# @uses: ZFS COMPRESSION_PROP CHECKSUM_PROP
849# @returns:
850#   0 if the dataset has been altered.
851#   1 if no pool name was passed in.
852#   2 if the dataset could not be found.
853#   3 if the dataset could not have it's properties set.
854#
855function dataset_set_defaultproperties
856{
857	typeset dataset="$1"
858
859	[[ -z $dataset ]] && return 1
860
861	typeset confset=
862	typeset -i found=0
863	for confset in $(zfs list); do
864		if [[ $dataset = $confset ]]; then
865			found=1
866			break
867		fi
868	done
869	[[ $found -eq 0 ]] && return 2
870	if [[ -n $COMPRESSION_PROP ]]; then
871		dataset_setprop $dataset compression $COMPRESSION_PROP || \
872			return 3
873		log_note "Compression set to '$COMPRESSION_PROP' on $dataset"
874	fi
875	if [[ -n $CHECKSUM_PROP ]]; then
876		dataset_setprop $dataset checksum $CHECKSUM_PROP || \
877			return 3
878		log_note "Checksum set to '$CHECKSUM_PROP' on $dataset"
879	fi
880	return 0
881}
882
883#
884# Check a numeric assertion
885# @parameter: $@ the assertion to check
886# @output: big loud notice if assertion failed
887# @use: log_fail
888#
889function assert
890{
891	(($@)) || log_fail "$@"
892}
893
894#
895# Function to format partition size of a disk
896# Given a disk cxtxdx reduces all partitions
897# to 0 size
898#
899function zero_partitions #<whole_disk_name>
900{
901	typeset diskname=$1
902	typeset i
903
904	if is_freebsd; then
905		gpart destroy -F $diskname
906	elif is_linux; then
907		DSK=$DEV_DSKDIR/$diskname
908		DSK=$(echo $DSK | sed -e "s|//|/|g")
909		log_must parted $DSK -s -- mklabel gpt
910		blockdev --rereadpt $DSK 2>/dev/null
911		block_device_wait
912	else
913		for i in 0 1 3 4 5 6 7
914		do
915			log_must set_partition $i "" 0mb $diskname
916		done
917	fi
918
919	return 0
920}
921
922#
923# Given a slice, size and disk, this function
924# formats the slice to the specified size.
925# Size should be specified with units as per
926# the `format` command requirements eg. 100mb 3gb
927#
928# NOTE: This entire interface is problematic for the Linux parted utility
929# which requires the end of the partition to be specified.  It would be
930# best to retire this interface and replace it with something more flexible.
931# At the moment a best effort is made.
932#
933# arguments: <slice_num> <slice_start> <size_plus_units>  <whole_disk_name>
934function set_partition
935{
936	typeset -i slicenum=$1
937	typeset start=$2
938	typeset size=$3
939	typeset disk=${4#$DEV_DSKDIR/}
940	disk=${disk#$DEV_RDSKDIR/}
941
942	case "$(uname)" in
943	Linux)
944		if [[ -z $size || -z $disk ]]; then
945			log_fail "The size or disk name is unspecified."
946		fi
947		disk=$DEV_DSKDIR/$disk
948		typeset size_mb=${size%%[mMgG]}
949
950		size_mb=${size_mb%%[mMgG][bB]}
951		if [[ ${size:1:1} == 'g' ]]; then
952			((size_mb = size_mb * 1024))
953		fi
954
955		# Create GPT partition table when setting slice 0 or
956		# when the device doesn't already contain a GPT label.
957		parted $disk -s -- print 1 >/dev/null
958		typeset ret_val=$?
959		if [[ $slicenum -eq 0 || $ret_val -ne 0 ]]; then
960			parted $disk -s -- mklabel gpt
961			if [[ $? -ne 0 ]]; then
962				log_note "Failed to create GPT partition table on $disk"
963				return 1
964			fi
965		fi
966
967		# When no start is given align on the first cylinder.
968		if [[ -z "$start" ]]; then
969			start=1
970		fi
971
972		# Determine the cylinder size for the device and using
973		# that calculate the end offset in cylinders.
974		typeset -i cly_size_kb=0
975		cly_size_kb=$(parted -m $disk -s -- \
976			unit cyl print | head -3 | tail -1 | \
977			awk -F '[:k.]' '{print $4}')
978		((end = (size_mb * 1024 / cly_size_kb) + start))
979
980		parted $disk -s -- \
981		    mkpart part$slicenum ${start}cyl ${end}cyl
982		typeset ret_val=$?
983		if [[ $ret_val -ne 0 ]]; then
984			log_note "Failed to create partition $slicenum on $disk"
985			return 1
986		fi
987
988		blockdev --rereadpt $disk 2>/dev/null
989		block_device_wait $disk
990		;;
991	FreeBSD)
992		if [[ -z $size || -z $disk ]]; then
993			log_fail "The size or disk name is unspecified."
994		fi
995		disk=$DEV_DSKDIR/$disk
996
997		if [[ $slicenum -eq 0 ]] || ! gpart show $disk >/dev/null 2>&1; then
998			gpart destroy -F $disk >/dev/null 2>&1
999			gpart create -s GPT $disk
1000			if [[ $? -ne 0 ]]; then
1001				log_note "Failed to create GPT partition table on $disk"
1002				return 1
1003			fi
1004		fi
1005
1006		typeset index=$((slicenum + 1))
1007
1008		if [[ -n $start ]]; then
1009			start="-b $start"
1010		fi
1011		gpart add -t freebsd-zfs $start -s $size -i $index $disk
1012		if [[ $ret_val -ne 0 ]]; then
1013			log_note "Failed to create partition $slicenum on $disk"
1014			return 1
1015		fi
1016
1017		block_device_wait $disk
1018		;;
1019	*)
1020		if [[ -z $slicenum || -z $size || -z $disk ]]; then
1021			log_fail "The slice, size or disk name is unspecified."
1022		fi
1023
1024		typeset format_file=/var/tmp/format_in.$$
1025
1026		echo "partition" >$format_file
1027		echo "$slicenum" >> $format_file
1028		echo "" >> $format_file
1029		echo "" >> $format_file
1030		echo "$start" >> $format_file
1031		echo "$size" >> $format_file
1032		echo "label" >> $format_file
1033		echo "" >> $format_file
1034		echo "q" >> $format_file
1035		echo "q" >> $format_file
1036
1037		format -e -s -d $disk -f $format_file
1038		typeset ret_val=$?
1039		rm -f $format_file
1040		;;
1041	esac
1042
1043	if [[ $ret_val -ne 0 ]]; then
1044		log_note "Unable to format $disk slice $slicenum to $size"
1045		return 1
1046	fi
1047	return 0
1048}
1049
1050#
1051# Delete all partitions on all disks - this is specifically for the use of multipath
1052# devices which currently can only be used in the test suite as raw/un-partitioned
1053# devices (ie a zpool cannot be created on a whole mpath device that has partitions)
1054#
1055function delete_partitions
1056{
1057	typeset disk
1058
1059	if [[ -z $DISKSARRAY ]]; then
1060		DISKSARRAY=$DISKS
1061	fi
1062
1063	if is_linux; then
1064		typeset -i part
1065		for disk in $DISKSARRAY; do
1066			for (( part = 1; part < MAX_PARTITIONS; part++ )); do
1067				typeset partition=${disk}${SLICE_PREFIX}${part}
1068				parted $DEV_DSKDIR/$disk -s rm $part > /dev/null 2>&1
1069				if lsblk | grep -qF ${partition}; then
1070					log_fail "Partition ${partition} not deleted"
1071				else
1072					log_note "Partition ${partition} deleted"
1073				fi
1074			done
1075		done
1076	elif is_freebsd; then
1077		for disk in $DISKSARRAY; do
1078			if gpart destroy -F $disk; then
1079				log_note "Partitions for ${disk} deleted"
1080			else
1081				log_fail "Partitions for ${disk} not deleted"
1082			fi
1083		done
1084	fi
1085}
1086
1087#
1088# Get the end cyl of the given slice
1089#
1090function get_endslice #<disk> <slice>
1091{
1092	typeset disk=$1
1093	typeset slice=$2
1094	if [[ -z $disk || -z $slice ]] ; then
1095		log_fail "The disk name or slice number is unspecified."
1096	fi
1097
1098	case "$(uname)" in
1099	Linux)
1100		endcyl=$(parted -s $DEV_DSKDIR/$disk -- unit cyl print | \
1101			awk "/part${slice}/"' {sub(/cyl/, "", $3); print $3}')
1102		((endcyl = (endcyl + 1)))
1103		;;
1104	FreeBSD)
1105		disk=${disk#/dev/zvol/}
1106		disk=${disk%p*}
1107		slice=$((slice + 1))
1108		endcyl=$(gpart show $disk | \
1109			awk -v slice=$slice '$3 == slice { print $1 + $2 }')
1110		;;
1111	*)
1112		disk=${disk#/dev/dsk/}
1113		disk=${disk#/dev/rdsk/}
1114		disk=${disk%s*}
1115
1116		typeset -i ratio=0
1117		ratio=$(prtvtoc /dev/rdsk/${disk}s2 | \
1118		    grep "sectors\/cylinder" | \
1119		    awk '{print $2}')
1120
1121		if ((ratio == 0)); then
1122			return
1123		fi
1124
1125		typeset -i endcyl=$(prtvtoc -h /dev/rdsk/${disk}s2 |
1126		    nawk -v token="$slice" '{if ($1==token) print $6}')
1127
1128		((endcyl = (endcyl + 1) / ratio))
1129		;;
1130	esac
1131
1132	echo $endcyl
1133}
1134
1135
1136#
1137# Given a size,disk and total slice number,  this function formats the
1138# disk slices from 0 to the total slice number with the same specified
1139# size.
1140#
1141function partition_disk	#<slice_size> <whole_disk_name>	<total_slices>
1142{
1143	typeset -i i=0
1144	typeset slice_size=$1
1145	typeset disk_name=$2
1146	typeset total_slices=$3
1147	typeset cyl
1148
1149	zero_partitions $disk_name
1150	while ((i < $total_slices)); do
1151		if ! is_linux; then
1152			if ((i == 2)); then
1153				((i = i + 1))
1154				continue
1155			fi
1156		fi
1157		log_must set_partition $i "$cyl" $slice_size $disk_name
1158		cyl=$(get_endslice $disk_name $i)
1159		((i = i+1))
1160	done
1161}
1162
1163#
1164# This function continues to write to a filenum number of files into dirnum
1165# number of directories until either file_write returns an error or the
1166# maximum number of files per directory have been written.
1167#
1168# Usage:
1169# fill_fs [destdir] [dirnum] [filenum] [bytes] [num_writes] [data]
1170#
1171# Return value: 0 on success
1172#		non 0 on error
1173#
1174# Where :
1175#	destdir:    is the directory where everything is to be created under
1176#	dirnum:	    the maximum number of subdirectories to use, -1 no limit
1177#	filenum:    the maximum number of files per subdirectory
1178#	bytes:	    number of bytes to write
1179#	num_writes: number of types to write out bytes
1180#	data:	    the data that will be written
1181#
1182#	E.g.
1183#	fill_fs /testdir 20 25 1024 256 0
1184#
1185# Note: bytes * num_writes equals the size of the testfile
1186#
1187function fill_fs # destdir dirnum filenum bytes num_writes data
1188{
1189	typeset destdir=${1:-$TESTDIR}
1190	typeset -i dirnum=${2:-50}
1191	typeset -i filenum=${3:-50}
1192	typeset -i bytes=${4:-8192}
1193	typeset -i num_writes=${5:-10240}
1194	typeset data=${6:-0}
1195
1196	mkdir -p $destdir/{1..$dirnum}
1197	for f in $destdir/{1..$dirnum}/$TESTFILE{1..$filenum}; do
1198		file_write -o create -f $f -b $bytes -c $num_writes -d $data \
1199		|| return $?
1200	done
1201	return 0
1202}
1203
1204#
1205# Simple function to get the specified property. If unable to
1206# get the property then exits.
1207#
1208# Note property is in 'parsable' format (-p)
1209#
1210function get_prop # property dataset
1211{
1212	typeset prop_val
1213	typeset prop=$1
1214	typeset dataset=$2
1215
1216	prop_val=$(zfs get -pH -o value $prop $dataset 2>/dev/null)
1217	if [[ $? -ne 0 ]]; then
1218		log_note "Unable to get $prop property for dataset " \
1219		"$dataset"
1220		return 1
1221	fi
1222
1223	echo "$prop_val"
1224	return 0
1225}
1226
1227#
1228# Simple function to get the specified property of pool. If unable to
1229# get the property then exits.
1230#
1231# Note property is in 'parsable' format (-p)
1232#
1233function get_pool_prop # property pool
1234{
1235	typeset prop_val
1236	typeset prop=$1
1237	typeset pool=$2
1238
1239	if poolexists $pool ; then
1240		prop_val=$(zpool get -pH $prop $pool 2>/dev/null | tail -1 | \
1241			awk '{print $3}')
1242		if [[ $? -ne 0 ]]; then
1243			log_note "Unable to get $prop property for pool " \
1244			"$pool"
1245			return 1
1246		fi
1247	else
1248		log_note "Pool $pool not exists."
1249		return 1
1250	fi
1251
1252	echo "$prop_val"
1253	return 0
1254}
1255
1256# Return 0 if a pool exists; $? otherwise
1257#
1258# $1 - pool name
1259
1260function poolexists
1261{
1262	typeset pool=$1
1263
1264	if [[ -z $pool ]]; then
1265		log_note "No pool name given."
1266		return 1
1267	fi
1268
1269	zpool get name "$pool" > /dev/null 2>&1
1270	return $?
1271}
1272
1273# Return 0 if all the specified datasets exist; $? otherwise
1274#
1275# $1-n  dataset name
1276function datasetexists
1277{
1278	if (($# == 0)); then
1279		log_note "No dataset name given."
1280		return 1
1281	fi
1282
1283	while (($# > 0)); do
1284		zfs get name $1 > /dev/null 2>&1 || \
1285			return $?
1286		shift
1287	done
1288
1289	return 0
1290}
1291
1292# return 0 if none of the specified datasets exists, otherwise return 1.
1293#
1294# $1-n  dataset name
1295function datasetnonexists
1296{
1297	if (($# == 0)); then
1298		log_note "No dataset name given."
1299		return 1
1300	fi
1301
1302	while (($# > 0)); do
1303		zfs list -H -t filesystem,snapshot,volume $1 > /dev/null 2>&1 \
1304		    && return 1
1305		shift
1306	done
1307
1308	return 0
1309}
1310
1311function is_shared_freebsd
1312{
1313	typeset fs=$1
1314
1315	pgrep -q mountd && showmount -E | grep -qx $fs
1316}
1317
1318function is_shared_illumos
1319{
1320	typeset fs=$1
1321	typeset mtpt
1322
1323	for mtpt in `share | awk '{print $2}'` ; do
1324		if [[ $mtpt == $fs ]] ; then
1325			return 0
1326		fi
1327	done
1328
1329	typeset stat=$(svcs -H -o STA nfs/server:default)
1330	if [[ $stat != "ON" ]]; then
1331		log_note "Current nfs/server status: $stat"
1332	fi
1333
1334	return 1
1335}
1336
1337function is_shared_linux
1338{
1339	typeset fs=$1
1340	typeset mtpt
1341
1342	for mtpt in `share | awk '{print $1}'` ; do
1343		if [[ $mtpt == $fs ]] ; then
1344			return 0
1345		fi
1346	done
1347	return 1
1348}
1349
1350#
1351# Given a mountpoint, or a dataset name, determine if it is shared via NFS.
1352#
1353# Returns 0 if shared, 1 otherwise.
1354#
1355function is_shared
1356{
1357	typeset fs=$1
1358	typeset mtpt
1359
1360	if [[ $fs != "/"* ]] ; then
1361		if datasetnonexists "$fs" ; then
1362			return 1
1363		else
1364			mtpt=$(get_prop mountpoint "$fs")
1365			case $mtpt in
1366				none|legacy|-) return 1
1367					;;
1368				*)	fs=$mtpt
1369					;;
1370			esac
1371		fi
1372	fi
1373
1374	case $(uname) in
1375	FreeBSD)	is_shared_freebsd "$fs"	;;
1376	Linux)		is_shared_linux "$fs"	;;
1377	*)		is_shared_illumos "$fs"	;;
1378	esac
1379}
1380
1381function is_exported_illumos
1382{
1383	typeset fs=$1
1384	typeset mtpt
1385
1386	for mtpt in `awk '{print $1}' /etc/dfs/sharetab` ; do
1387		if [[ $mtpt == $fs ]] ; then
1388			return 0
1389		fi
1390	done
1391
1392	return 1
1393}
1394
1395function is_exported_freebsd
1396{
1397	typeset fs=$1
1398	typeset mtpt
1399
1400	for mtpt in `awk '{print $1}' /etc/zfs/exports` ; do
1401		if [[ $mtpt == $fs ]] ; then
1402			return 0
1403		fi
1404	done
1405
1406	return 1
1407}
1408
1409function is_exported_linux
1410{
1411	typeset fs=$1
1412	typeset mtpt
1413
1414	for mtpt in `awk '{print $1}' /etc/exports.d/zfs.exports` ; do
1415		if [[ $mtpt == $fs ]] ; then
1416			return 0
1417		fi
1418	done
1419
1420	return 1
1421}
1422
1423#
1424# Given a mountpoint, or a dataset name, determine if it is exported via
1425# the os-specific NFS exports file.
1426#
1427# Returns 0 if exported, 1 otherwise.
1428#
1429function is_exported
1430{
1431	typeset fs=$1
1432	typeset mtpt
1433
1434	if [[ $fs != "/"* ]] ; then
1435		if datasetnonexists "$fs" ; then
1436			return 1
1437		else
1438			mtpt=$(get_prop mountpoint "$fs")
1439			case $mtpt in
1440				none|legacy|-) return 1
1441					;;
1442				*)	fs=$mtpt
1443					;;
1444			esac
1445		fi
1446	fi
1447
1448	case $(uname) in
1449	FreeBSD)	is_exported_freebsd "$fs"	;;
1450	Linux)		is_exported_linux "$fs"	;;
1451	*)		is_exported_illumos "$fs"	;;
1452	esac
1453}
1454
1455#
1456# Given a dataset name determine if it is shared via SMB.
1457#
1458# Returns 0 if shared, 1 otherwise.
1459#
1460function is_shared_smb
1461{
1462	typeset fs=$1
1463	typeset mtpt
1464
1465	if datasetnonexists "$fs" ; then
1466		return 1
1467	else
1468		fs=$(echo $fs | tr / _)
1469	fi
1470
1471	if is_linux; then
1472		for mtpt in `net usershare list | awk '{print $1}'` ; do
1473			if [[ $mtpt == $fs ]] ; then
1474				return 0
1475			fi
1476		done
1477		return 1
1478	else
1479		log_note "Currently unsupported by the test framework"
1480		return 1
1481	fi
1482}
1483
1484#
1485# Given a mountpoint, determine if it is not shared via NFS.
1486#
1487# Returns 0 if not shared, 1 otherwise.
1488#
1489function not_shared
1490{
1491	typeset fs=$1
1492
1493	is_shared $fs
1494	if (($? == 0)); then
1495		return 1
1496	fi
1497
1498	return 0
1499}
1500
1501#
1502# Given a dataset determine if it is not shared via SMB.
1503#
1504# Returns 0 if not shared, 1 otherwise.
1505#
1506function not_shared_smb
1507{
1508	typeset fs=$1
1509
1510	is_shared_smb $fs
1511	if (($? == 0)); then
1512		return 1
1513	fi
1514
1515	return 0
1516}
1517
1518#
1519# Helper function to unshare a mountpoint.
1520#
1521function unshare_fs #fs
1522{
1523	typeset fs=$1
1524
1525	is_shared $fs || is_shared_smb $fs
1526	if (($? == 0)); then
1527		zfs unshare $fs || log_fail "zfs unshare $fs failed"
1528	fi
1529
1530	return 0
1531}
1532
1533#
1534# Helper function to share a NFS mountpoint.
1535#
1536function share_nfs #fs
1537{
1538	typeset fs=$1
1539
1540	if is_linux; then
1541		is_shared $fs
1542		if (($? != 0)); then
1543			log_must share "*:$fs"
1544		fi
1545	else
1546		is_shared $fs
1547		if (($? != 0)); then
1548			log_must share -F nfs $fs
1549		fi
1550	fi
1551
1552	return 0
1553}
1554
1555#
1556# Helper function to unshare a NFS mountpoint.
1557#
1558function unshare_nfs #fs
1559{
1560	typeset fs=$1
1561
1562	if is_linux; then
1563		is_shared $fs
1564		if (($? == 0)); then
1565			log_must unshare -u "*:$fs"
1566		fi
1567	else
1568		is_shared $fs
1569		if (($? == 0)); then
1570			log_must unshare -F nfs $fs
1571		fi
1572	fi
1573
1574	return 0
1575}
1576
1577#
1578# Helper function to show NFS shares.
1579#
1580function showshares_nfs
1581{
1582	if is_linux; then
1583		share -v
1584	else
1585		share -F nfs
1586	fi
1587
1588	return 0
1589}
1590
1591#
1592# Helper function to show SMB shares.
1593#
1594function showshares_smb
1595{
1596	if is_linux; then
1597		net usershare list
1598	else
1599		share -F smb
1600	fi
1601
1602	return 0
1603}
1604
1605function check_nfs
1606{
1607	if is_linux; then
1608		share -s
1609	elif is_freebsd; then
1610		showmount -e
1611	else
1612		log_unsupported "Unknown platform"
1613	fi
1614
1615	if [[ $? -ne 0 ]]; then
1616		log_unsupported "The NFS utilities are not installed"
1617	fi
1618}
1619
1620#
1621# Check NFS server status and trigger it online.
1622#
1623function setup_nfs_server
1624{
1625	# Cannot share directory in non-global zone.
1626	#
1627	if ! is_global_zone; then
1628		log_note "Cannot trigger NFS server by sharing in LZ."
1629		return
1630	fi
1631
1632	if is_linux; then
1633		#
1634		# Re-synchronize /var/lib/nfs/etab with /etc/exports and
1635		# /etc/exports.d./* to provide a clean test environment.
1636		#
1637		log_must share -r
1638
1639		log_note "NFS server must be started prior to running ZTS."
1640		return
1641	elif is_freebsd; then
1642		kill -s HUP $(cat /var/run/mountd.pid)
1643
1644		log_note "NFS server must be started prior to running ZTS."
1645		return
1646	fi
1647
1648	typeset nfs_fmri="svc:/network/nfs/server:default"
1649	if [[ $(svcs -Ho STA $nfs_fmri) != "ON" ]]; then
1650		#
1651		# Only really sharing operation can enable NFS server
1652		# to online permanently.
1653		#
1654		typeset dummy=/tmp/dummy
1655
1656		if [[ -d $dummy ]]; then
1657			log_must rm -rf $dummy
1658		fi
1659
1660		log_must mkdir $dummy
1661		log_must share $dummy
1662
1663		#
1664		# Waiting for fmri's status to be the final status.
1665		# Otherwise, in transition, an asterisk (*) is appended for
1666		# instances, unshare will reverse status to 'DIS' again.
1667		#
1668		# Waiting for 1's at least.
1669		#
1670		log_must sleep 1
1671		timeout=10
1672		while [[ timeout -ne 0 && $(svcs -Ho STA $nfs_fmri) == *'*' ]]
1673		do
1674			log_must sleep 1
1675
1676			((timeout -= 1))
1677		done
1678
1679		log_must unshare $dummy
1680		log_must rm -rf $dummy
1681	fi
1682
1683	log_note "Current NFS status: '$(svcs -Ho STA,FMRI $nfs_fmri)'"
1684}
1685
1686#
1687# To verify whether calling process is in global zone
1688#
1689# Return 0 if in global zone, 1 in non-global zone
1690#
1691function is_global_zone
1692{
1693	if is_linux || is_freebsd; then
1694		return 0
1695	else
1696		typeset cur_zone=$(zonename 2>/dev/null)
1697		if [[ $cur_zone != "global" ]]; then
1698			return 1
1699		fi
1700		return 0
1701	fi
1702}
1703
1704#
1705# Verify whether test is permitted to run from
1706# global zone, local zone, or both
1707#
1708# $1 zone limit, could be "global", "local", or "both"(no limit)
1709#
1710# Return 0 if permitted, otherwise exit with log_unsupported
1711#
1712function verify_runnable # zone limit
1713{
1714	typeset limit=$1
1715
1716	[[ -z $limit ]] && return 0
1717
1718	if is_global_zone ; then
1719		case $limit in
1720			global|both)
1721				;;
1722			local)	log_unsupported "Test is unable to run from "\
1723					"global zone."
1724				;;
1725			*)	log_note "Warning: unknown limit $limit - " \
1726					"use both."
1727				;;
1728		esac
1729	else
1730		case $limit in
1731			local|both)
1732				;;
1733			global)	log_unsupported "Test is unable to run from "\
1734					"local zone."
1735				;;
1736			*)	log_note "Warning: unknown limit $limit - " \
1737					"use both."
1738				;;
1739		esac
1740
1741		reexport_pool
1742	fi
1743
1744	return 0
1745}
1746
1747# Return 0 if create successfully or the pool exists; $? otherwise
1748# Note: In local zones, this function should return 0 silently.
1749#
1750# $1 - pool name
1751# $2-n - [keyword] devs_list
1752
1753function create_pool #pool devs_list
1754{
1755	typeset pool=${1%%/*}
1756
1757	shift
1758
1759	if [[ -z $pool ]]; then
1760		log_note "Missing pool name."
1761		return 1
1762	fi
1763
1764	if poolexists $pool ; then
1765		destroy_pool $pool
1766	fi
1767
1768	if is_global_zone ; then
1769		[[ -d /$pool ]] && rm -rf /$pool
1770		log_must zpool create -f $pool $@
1771	fi
1772
1773	return 0
1774}
1775
1776# Return 0 if destroy successfully or the pool exists; $? otherwise
1777# Note: In local zones, this function should return 0 silently.
1778#
1779# $1 - pool name
1780# Destroy pool with the given parameters.
1781
1782function destroy_pool #pool
1783{
1784	typeset pool=${1%%/*}
1785	typeset mtpt
1786
1787	if [[ -z $pool ]]; then
1788		log_note "No pool name given."
1789		return 1
1790	fi
1791
1792	if is_global_zone ; then
1793		if poolexists "$pool" ; then
1794			mtpt=$(get_prop mountpoint "$pool")
1795
1796			# At times, syseventd/udev activity can cause attempts
1797			# to destroy a pool to fail with EBUSY. We retry a few
1798			# times allowing failures before requiring the destroy
1799			# to succeed.
1800			log_must_busy zpool destroy -f $pool
1801
1802			[[ -d $mtpt ]] && \
1803				log_must rm -rf $mtpt
1804		else
1805			log_note "Pool does not exist. ($pool)"
1806			return 1
1807		fi
1808	fi
1809
1810	return 0
1811}
1812
1813# Return 0 if created successfully; $? otherwise
1814#
1815# $1 - dataset name
1816# $2-n - dataset options
1817
1818function create_dataset #dataset dataset_options
1819{
1820	typeset dataset=$1
1821
1822	shift
1823
1824	if [[ -z $dataset ]]; then
1825		log_note "Missing dataset name."
1826		return 1
1827	fi
1828
1829	if datasetexists $dataset ; then
1830		destroy_dataset $dataset
1831	fi
1832
1833	log_must zfs create $@ $dataset
1834
1835	return 0
1836}
1837
1838# Return 0 if destroy successfully or the dataset exists; $? otherwise
1839# Note: In local zones, this function should return 0 silently.
1840#
1841# $1 - dataset name
1842# $2 - custom arguments for zfs destroy
1843# Destroy dataset with the given parameters.
1844
1845function destroy_dataset #dataset #args
1846{
1847	typeset dataset=$1
1848	typeset mtpt
1849	typeset args=${2:-""}
1850
1851	if [[ -z $dataset ]]; then
1852		log_note "No dataset name given."
1853		return 1
1854	fi
1855
1856	if is_global_zone ; then
1857		if datasetexists "$dataset" ; then
1858			mtpt=$(get_prop mountpoint "$dataset")
1859			log_must_busy zfs destroy $args $dataset
1860
1861			[[ -d $mtpt ]] && \
1862				log_must rm -rf $mtpt
1863		else
1864			log_note "Dataset does not exist. ($dataset)"
1865			return 1
1866		fi
1867	fi
1868
1869	return 0
1870}
1871
1872#
1873# Firstly, create a pool with 5 datasets. Then, create a single zone and
1874# export the 5 datasets to it. In addition, we also add a ZFS filesystem
1875# and a zvol device to the zone.
1876#
1877# $1 zone name
1878# $2 zone root directory prefix
1879# $3 zone ip
1880#
1881function zfs_zones_setup #zone_name zone_root zone_ip
1882{
1883	typeset zone_name=${1:-$(hostname)-z}
1884	typeset zone_root=${2:-"/zone_root"}
1885	typeset zone_ip=${3:-"10.1.1.10"}
1886	typeset prefix_ctr=$ZONE_CTR
1887	typeset pool_name=$ZONE_POOL
1888	typeset -i cntctr=5
1889	typeset -i i=0
1890
1891	# Create pool and 5 container within it
1892	#
1893	[[ -d /$pool_name ]] && rm -rf /$pool_name
1894	log_must zpool create -f $pool_name $DISKS
1895	while ((i < cntctr)); do
1896		log_must zfs create $pool_name/$prefix_ctr$i
1897		((i += 1))
1898	done
1899
1900	# create a zvol
1901	log_must zfs create -V 1g $pool_name/zone_zvol
1902	block_device_wait
1903
1904	#
1905	# If current system support slog, add slog device for pool
1906	#
1907	if verify_slog_support ; then
1908		typeset sdevs="$TEST_BASE_DIR/sdev1 $TEST_BASE_DIR/sdev2"
1909		log_must mkfile $MINVDEVSIZE $sdevs
1910		log_must zpool add $pool_name log mirror $sdevs
1911	fi
1912
1913	# this isn't supported just yet.
1914	# Create a filesystem. In order to add this to
1915	# the zone, it must have it's mountpoint set to 'legacy'
1916	# log_must zfs create $pool_name/zfs_filesystem
1917	# log_must zfs set mountpoint=legacy $pool_name/zfs_filesystem
1918
1919	[[ -d $zone_root ]] && \
1920		log_must rm -rf $zone_root/$zone_name
1921	[[ ! -d $zone_root ]] && \
1922		log_must mkdir -p -m 0700 $zone_root/$zone_name
1923
1924	# Create zone configure file and configure the zone
1925	#
1926	typeset zone_conf=/tmp/zone_conf.$$
1927	echo "create" > $zone_conf
1928	echo "set zonepath=$zone_root/$zone_name" >> $zone_conf
1929	echo "set autoboot=true" >> $zone_conf
1930	i=0
1931	while ((i < cntctr)); do
1932		echo "add dataset" >> $zone_conf
1933		echo "set name=$pool_name/$prefix_ctr$i" >> \
1934			$zone_conf
1935		echo "end" >> $zone_conf
1936		((i += 1))
1937	done
1938
1939	# add our zvol to the zone
1940	echo "add device" >> $zone_conf
1941	echo "set match=/dev/zvol/dsk/$pool_name/zone_zvol" >> $zone_conf
1942	echo "end" >> $zone_conf
1943
1944	# add a corresponding zvol rdsk to the zone
1945	echo "add device" >> $zone_conf
1946	echo "set match=$ZVOL_RDEVDIR/$pool_name/zone_zvol" >> $zone_conf
1947	echo "end" >> $zone_conf
1948
1949	# once it's supported, we'll add our filesystem to the zone
1950	# echo "add fs" >> $zone_conf
1951	# echo "set type=zfs" >> $zone_conf
1952	# echo "set special=$pool_name/zfs_filesystem" >> $zone_conf
1953	# echo "set dir=/export/zfs_filesystem" >> $zone_conf
1954	# echo "end" >> $zone_conf
1955
1956	echo "verify" >> $zone_conf
1957	echo "commit" >> $zone_conf
1958	log_must zonecfg -z $zone_name -f $zone_conf
1959	log_must rm -f $zone_conf
1960
1961	# Install the zone
1962	zoneadm -z $zone_name install
1963	if (($? == 0)); then
1964		log_note "SUCCESS: zoneadm -z $zone_name install"
1965	else
1966		log_fail "FAIL: zoneadm -z $zone_name install"
1967	fi
1968
1969	# Install sysidcfg file
1970	#
1971	typeset sysidcfg=$zone_root/$zone_name/root/etc/sysidcfg
1972	echo "system_locale=C" > $sysidcfg
1973	echo  "terminal=dtterm" >> $sysidcfg
1974	echo  "network_interface=primary {" >> $sysidcfg
1975	echo  "hostname=$zone_name" >> $sysidcfg
1976	echo  "}" >> $sysidcfg
1977	echo  "name_service=NONE" >> $sysidcfg
1978	echo  "root_password=mo791xfZ/SFiw" >> $sysidcfg
1979	echo  "security_policy=NONE" >> $sysidcfg
1980	echo  "timezone=US/Eastern" >> $sysidcfg
1981
1982	# Boot this zone
1983	log_must zoneadm -z $zone_name boot
1984}
1985
1986#
1987# Reexport TESTPOOL & TESTPOOL(1-4)
1988#
1989function reexport_pool
1990{
1991	typeset -i cntctr=5
1992	typeset -i i=0
1993
1994	while ((i < cntctr)); do
1995		if ((i == 0)); then
1996			TESTPOOL=$ZONE_POOL/$ZONE_CTR$i
1997			if ! ismounted $TESTPOOL; then
1998				log_must zfs mount $TESTPOOL
1999			fi
2000		else
2001			eval TESTPOOL$i=$ZONE_POOL/$ZONE_CTR$i
2002			if eval ! ismounted \$TESTPOOL$i; then
2003				log_must eval zfs mount \$TESTPOOL$i
2004			fi
2005		fi
2006		((i += 1))
2007	done
2008}
2009
2010#
2011# Verify a given disk or pool state
2012#
2013# Return 0 is pool/disk matches expected state, 1 otherwise
2014#
2015function check_state # pool disk state{online,offline,degraded}
2016{
2017	typeset pool=$1
2018	typeset disk=${2#$DEV_DSKDIR/}
2019	typeset state=$3
2020
2021	[[ -z $pool ]] || [[ -z $state ]] \
2022	    && log_fail "Arguments invalid or missing"
2023
2024	if [[ -z $disk ]]; then
2025		#check pool state only
2026		zpool get -H -o value health $pool \
2027		    | grep -i "$state" > /dev/null 2>&1
2028	else
2029		zpool status -v $pool | grep "$disk"  \
2030		    | grep -i "$state" > /dev/null 2>&1
2031	fi
2032
2033	return $?
2034}
2035
2036#
2037# Get the mountpoint of snapshot
2038# For the snapshot use <mp_filesystem>/.zfs/snapshot/<snap>
2039# as its mountpoint
2040#
2041function snapshot_mountpoint
2042{
2043	typeset dataset=${1:-$TESTPOOL/$TESTFS@$TESTSNAP}
2044
2045	if [[ $dataset != *@* ]]; then
2046		log_fail "Error name of snapshot '$dataset'."
2047	fi
2048
2049	typeset fs=${dataset%@*}
2050	typeset snap=${dataset#*@}
2051
2052	if [[ -z $fs || -z $snap ]]; then
2053		log_fail "Error name of snapshot '$dataset'."
2054	fi
2055
2056	echo $(get_prop mountpoint $fs)/.zfs/snapshot/$snap
2057}
2058
2059#
2060# Given a device and 'ashift' value verify it's correctly set on every label
2061#
2062function verify_ashift # device ashift
2063{
2064	typeset device="$1"
2065	typeset ashift="$2"
2066
2067	zdb -e -lll $device | awk -v ashift=$ashift '/ashift: / {
2068	    if (ashift != $2)
2069	        exit 1;
2070	    else
2071	        count++;
2072	    } END {
2073	    if (count != 4)
2074	        exit 1;
2075	    else
2076	        exit 0;
2077	    }'
2078
2079	return $?
2080}
2081
2082#
2083# Given a pool and file system, this function will verify the file system
2084# using the zdb internal tool. Note that the pool is exported and imported
2085# to ensure it has consistent state.
2086#
2087function verify_filesys # pool filesystem dir
2088{
2089	typeset pool="$1"
2090	typeset filesys="$2"
2091	typeset zdbout="/tmp/zdbout.$$"
2092
2093	shift
2094	shift
2095	typeset dirs=$@
2096	typeset search_path=""
2097
2098	log_note "Calling zdb to verify filesystem '$filesys'"
2099	zfs unmount -a > /dev/null 2>&1
2100	log_must zpool export $pool
2101
2102	if [[ -n $dirs ]] ; then
2103		for dir in $dirs ; do
2104			search_path="$search_path -d $dir"
2105		done
2106	fi
2107
2108	log_must zpool import $search_path $pool
2109
2110	zdb -cudi $filesys > $zdbout 2>&1
2111	if [[ $? != 0 ]]; then
2112		log_note "Output: zdb -cudi $filesys"
2113		cat $zdbout
2114		log_fail "zdb detected errors with: '$filesys'"
2115	fi
2116
2117	log_must zfs mount -a
2118	log_must rm -rf $zdbout
2119}
2120
2121#
2122# Given a pool issue a scrub and verify that no checksum errors are reported.
2123#
2124function verify_pool
2125{
2126	typeset pool=${1:-$TESTPOOL}
2127
2128	log_must zpool scrub $pool
2129	log_must wait_scrubbed $pool
2130
2131	typeset -i cksum=$(zpool status $pool | awk '
2132	    !NF { isvdev = 0 }
2133	    isvdev { errors += $NF }
2134	    /CKSUM$/ { isvdev = 1 }
2135	    END { print errors }
2136	')
2137	if [[ $cksum != 0 ]]; then
2138		log_must zpool status -v
2139	        log_fail "Unexpected CKSUM errors found on $pool ($cksum)"
2140	fi
2141}
2142
2143#
2144# Given a pool, and this function list all disks in the pool
2145#
2146function get_disklist # pool
2147{
2148	typeset disklist=""
2149
2150	disklist=$(zpool iostat -v $1 | nawk '(NR >4) {print $1}' | \
2151	    grep -v "\-\-\-\-\-" | \
2152	    grep -vEe "^(mirror|raidz[1-3]|draid[1-3]|spare|log|cache|special|dedup)|\-[0-9]$")
2153
2154	echo $disklist
2155}
2156
2157#
2158# Given a pool, and this function list all disks in the pool with their full
2159# path (like "/dev/sda" instead of "sda").
2160#
2161function get_disklist_fullpath # pool
2162{
2163	args="-P $1"
2164	get_disklist $args
2165}
2166
2167
2168
2169# /**
2170#  This function kills a given list of processes after a time period. We use
2171#  this in the stress tests instead of STF_TIMEOUT so that we can have processes
2172#  run for a fixed amount of time, yet still pass. Tests that hit STF_TIMEOUT
2173#  would be listed as FAIL, which we don't want : we're happy with stress tests
2174#  running for a certain amount of time, then finishing.
2175#
2176# @param $1 the time in seconds after which we should terminate these processes
2177# @param $2..$n the processes we wish to terminate.
2178# */
2179function stress_timeout
2180{
2181	typeset -i TIMEOUT=$1
2182	shift
2183	typeset cpids="$@"
2184
2185	log_note "Waiting for child processes($cpids). " \
2186		"It could last dozens of minutes, please be patient ..."
2187	log_must sleep $TIMEOUT
2188
2189	log_note "Killing child processes after ${TIMEOUT} stress timeout."
2190	typeset pid
2191	for pid in $cpids; do
2192		ps -p $pid > /dev/null 2>&1
2193		if (($? == 0)); then
2194			log_must kill -USR1 $pid
2195		fi
2196	done
2197}
2198
2199#
2200# Verify a given hotspare disk is inuse or avail
2201#
2202# Return 0 is pool/disk matches expected state, 1 otherwise
2203#
2204function check_hotspare_state # pool disk state{inuse,avail}
2205{
2206	typeset pool=$1
2207	typeset disk=${2#$DEV_DSKDIR/}
2208	typeset state=$3
2209
2210	cur_state=$(get_device_state $pool $disk "spares")
2211
2212	if [[ $state != ${cur_state} ]]; then
2213		return 1
2214	fi
2215	return 0
2216}
2217
2218#
2219# Wait until a hotspare transitions to a given state or times out.
2220#
2221# Return 0 when  pool/disk matches expected state, 1 on timeout.
2222#
2223function wait_hotspare_state # pool disk state timeout
2224{
2225	typeset pool=$1
2226	typeset disk=${2#*$DEV_DSKDIR/}
2227	typeset state=$3
2228	typeset timeout=${4:-60}
2229	typeset -i i=0
2230
2231	while [[ $i -lt $timeout ]]; do
2232		if check_hotspare_state $pool $disk $state; then
2233			return 0
2234		fi
2235
2236		i=$((i+1))
2237		sleep 1
2238	done
2239
2240	return 1
2241}
2242
2243#
2244# Verify a given slog disk is inuse or avail
2245#
2246# Return 0 is pool/disk matches expected state, 1 otherwise
2247#
2248function check_slog_state # pool disk state{online,offline,unavail}
2249{
2250	typeset pool=$1
2251	typeset disk=${2#$DEV_DSKDIR/}
2252	typeset state=$3
2253
2254	cur_state=$(get_device_state $pool $disk "logs")
2255
2256	if [[ $state != ${cur_state} ]]; then
2257		return 1
2258	fi
2259	return 0
2260}
2261
2262#
2263# Verify a given vdev disk is inuse or avail
2264#
2265# Return 0 is pool/disk matches expected state, 1 otherwise
2266#
2267function check_vdev_state # pool disk state{online,offline,unavail,removed}
2268{
2269	typeset pool=$1
2270	typeset disk=${2#*$DEV_DSKDIR/}
2271	typeset state=$3
2272
2273	cur_state=$(get_device_state $pool $disk)
2274
2275	if [[ $state != ${cur_state} ]]; then
2276		return 1
2277	fi
2278	return 0
2279}
2280
2281#
2282# Wait until a vdev transitions to a given state or times out.
2283#
2284# Return 0 when  pool/disk matches expected state, 1 on timeout.
2285#
2286function wait_vdev_state # pool disk state timeout
2287{
2288	typeset pool=$1
2289	typeset disk=${2#*$DEV_DSKDIR/}
2290	typeset state=$3
2291	typeset timeout=${4:-60}
2292	typeset -i i=0
2293
2294	while [[ $i -lt $timeout ]]; do
2295		if check_vdev_state $pool $disk $state; then
2296			return 0
2297		fi
2298
2299		i=$((i+1))
2300		sleep 1
2301	done
2302
2303	return 1
2304}
2305
2306#
2307# Check the output of 'zpool status -v <pool>',
2308# and to see if the content of <token> contain the <keyword> specified.
2309#
2310# Return 0 is contain, 1 otherwise
2311#
2312function check_pool_status # pool token keyword <verbose>
2313{
2314	typeset pool=$1
2315	typeset token=$2
2316	typeset keyword=$3
2317	typeset verbose=${4:-false}
2318
2319	scan=$(zpool status -v "$pool" 2>/dev/null | nawk -v token="$token:" '
2320		($1==token) {print $0}')
2321	if [[ $verbose == true ]]; then
2322		log_note $scan
2323	fi
2324	echo $scan | grep -qi "$keyword"
2325	return $?
2326}
2327
2328#
2329# The following functions are instance of check_pool_status()
2330#	is_pool_resilvering - to check if the pool resilver is in progress
2331#	is_pool_resilvered - to check if the pool resilver is completed
2332#	is_pool_scrubbing - to check if the pool scrub is in progress
2333#	is_pool_scrubbed - to check if the pool scrub is completed
2334#	is_pool_scrub_stopped - to check if the pool scrub is stopped
2335#	is_pool_scrub_paused - to check if the pool scrub has paused
2336#	is_pool_removing - to check if the pool removing is a vdev
2337#	is_pool_removed - to check if the pool remove is completed
2338#	is_pool_discarding - to check if the pool checkpoint is being discarded
2339#
2340function is_pool_resilvering #pool <verbose>
2341{
2342	check_pool_status "$1" "scan" \
2343	    "resilver[ ()0-9A-Za-z:_-]* in progress since" $2
2344	return $?
2345}
2346
2347function is_pool_resilvered #pool <verbose>
2348{
2349	check_pool_status "$1" "scan" "resilvered " $2
2350	return $?
2351}
2352
2353function is_pool_scrubbing #pool <verbose>
2354{
2355	check_pool_status "$1" "scan" "scrub in progress since " $2
2356	return $?
2357}
2358
2359function is_pool_scrubbed #pool <verbose>
2360{
2361	check_pool_status "$1" "scan" "scrub repaired" $2
2362	return $?
2363}
2364
2365function is_pool_scrub_stopped #pool <verbose>
2366{
2367	check_pool_status "$1" "scan" "scrub canceled" $2
2368	return $?
2369}
2370
2371function is_pool_scrub_paused #pool <verbose>
2372{
2373	check_pool_status "$1" "scan" "scrub paused since " $2
2374	return $?
2375}
2376
2377function is_pool_removing #pool
2378{
2379	check_pool_status "$1" "remove" "in progress since "
2380	return $?
2381}
2382
2383function is_pool_removed #pool
2384{
2385	check_pool_status "$1" "remove" "completed on"
2386	return $?
2387}
2388
2389function is_pool_discarding #pool
2390{
2391	check_pool_status "$1" "checkpoint" "discarding"
2392	return $?
2393}
2394
2395function wait_for_degraded
2396{
2397	typeset pool=$1
2398	typeset timeout=${2:-30}
2399	typeset t0=$SECONDS
2400
2401	while :; do
2402		[[ $(get_pool_prop health $pool) == "DEGRADED" ]] && break
2403		log_note "$pool is not yet degraded."
2404		sleep 1
2405		if ((SECONDS - t0 > $timeout)); then
2406			log_note "$pool not degraded after $timeout seconds."
2407			return 1
2408		fi
2409	done
2410
2411	return 0
2412}
2413
2414#
2415# Use create_pool()/destroy_pool() to clean up the information in
2416# in the given disk to avoid slice overlapping.
2417#
2418function cleanup_devices #vdevs
2419{
2420	typeset pool="foopool$$"
2421
2422	for vdev in $@; do
2423		zero_partitions $vdev
2424	done
2425
2426	poolexists $pool && destroy_pool $pool
2427	create_pool $pool $@
2428	destroy_pool $pool
2429
2430	return 0
2431}
2432
2433#/**
2434# A function to find and locate free disks on a system or from given
2435# disks as the parameter. It works by locating disks that are in use
2436# as swap devices and dump devices, and also disks listed in /etc/vfstab
2437#
2438# $@ given disks to find which are free, default is all disks in
2439# the test system
2440#
2441# @return a string containing the list of available disks
2442#*/
2443function find_disks
2444{
2445	# Trust provided list, no attempt is made to locate unused devices.
2446	if is_linux || is_freebsd; then
2447		echo "$@"
2448		return
2449	fi
2450
2451
2452	sfi=/tmp/swaplist.$$
2453	dmpi=/tmp/dumpdev.$$
2454	max_finddisksnum=${MAX_FINDDISKSNUM:-6}
2455
2456	swap -l > $sfi
2457	dumpadm > $dmpi 2>/dev/null
2458
2459# write an awk script that can process the output of format
2460# to produce a list of disks we know about. Note that we have
2461# to escape "$2" so that the shell doesn't interpret it while
2462# we're creating the awk script.
2463# -------------------
2464	cat > /tmp/find_disks.awk <<EOF
2465#!/bin/nawk -f
2466	BEGIN { FS="."; }
2467
2468	/^Specify disk/{
2469		searchdisks=0;
2470	}
2471
2472	{
2473		if (searchdisks && \$2 !~ "^$"){
2474			split(\$2,arr," ");
2475			print arr[1];
2476		}
2477	}
2478
2479	/^AVAILABLE DISK SELECTIONS:/{
2480		searchdisks=1;
2481	}
2482EOF
2483#---------------------
2484
2485	chmod 755 /tmp/find_disks.awk
2486	disks=${@:-$(echo "" | format -e 2>/dev/null | /tmp/find_disks.awk)}
2487	rm /tmp/find_disks.awk
2488
2489	unused=""
2490	for disk in $disks; do
2491	# Check for mounted
2492		grep "${disk}[sp]" /etc/mnttab >/dev/null
2493		(($? == 0)) && continue
2494	# Check for swap
2495		grep "${disk}[sp]" $sfi >/dev/null
2496		(($? == 0)) && continue
2497	# check for dump device
2498		grep "${disk}[sp]" $dmpi >/dev/null
2499		(($? == 0)) && continue
2500	# check to see if this disk hasn't been explicitly excluded
2501	# by a user-set environment variable
2502		echo "${ZFS_HOST_DEVICES_IGNORE}" | grep "${disk}" > /dev/null
2503		(($? == 0)) && continue
2504		unused_candidates="$unused_candidates $disk"
2505	done
2506	rm $sfi
2507	rm $dmpi
2508
2509# now just check to see if those disks do actually exist
2510# by looking for a device pointing to the first slice in
2511# each case. limit the number to max_finddisksnum
2512	count=0
2513	for disk in $unused_candidates; do
2514		if is_disk_device $DEV_DSKDIR/${disk}s0 && \
2515		    [ $count -lt $max_finddisksnum ]; then
2516			unused="$unused $disk"
2517			# do not impose limit if $@ is provided
2518			[[ -z $@ ]] && ((count = count + 1))
2519		fi
2520	done
2521
2522# finally, return our disk list
2523	echo $unused
2524}
2525
2526function add_user_freebsd #<group_name> <user_name> <basedir>
2527{
2528	typeset group=$1
2529	typeset user=$2
2530	typeset basedir=$3
2531
2532	# Check to see if the user exists.
2533	if id $user > /dev/null 2>&1; then
2534		return 0
2535	fi
2536
2537	# Assign 1000 as the base uid
2538	typeset -i uid=1000
2539	while true; do
2540		typeset -i ret
2541		pw useradd -u $uid -g $group -d $basedir/$user -m -n $user
2542		ret=$?
2543		case $ret in
2544			0) break ;;
2545			# The uid is not unique
2546			65) ((uid += 1)) ;;
2547			*) return 1 ;;
2548		esac
2549		if [[ $uid == 65000 ]]; then
2550			log_fail "No user id available under 65000 for $user"
2551		fi
2552	done
2553
2554	# Silence MOTD
2555	touch $basedir/$user/.hushlogin
2556
2557	return 0
2558}
2559
2560#
2561# Delete the specified user.
2562#
2563# $1 login name
2564#
2565function del_user_freebsd #<logname>
2566{
2567	typeset user=$1
2568
2569	if id $user > /dev/null 2>&1; then
2570		log_must pw userdel $user
2571	fi
2572
2573	return 0
2574}
2575
2576#
2577# Select valid gid and create specified group.
2578#
2579# $1 group name
2580#
2581function add_group_freebsd #<group_name>
2582{
2583	typeset group=$1
2584
2585	# See if the group already exists.
2586	if pw groupshow $group >/dev/null 2>&1; then
2587		return 0
2588	fi
2589
2590	# Assign 1000 as the base gid
2591	typeset -i gid=1000
2592	while true; do
2593		pw groupadd -g $gid -n $group > /dev/null 2>&1
2594		typeset -i ret=$?
2595		case $ret in
2596			0) return 0 ;;
2597			# The gid is not  unique
2598			65) ((gid += 1)) ;;
2599			*) return 1 ;;
2600		esac
2601		if [[ $gid == 65000 ]]; then
2602			log_fail "No user id available under 65000 for $group"
2603		fi
2604	done
2605}
2606
2607#
2608# Delete the specified group.
2609#
2610# $1 group name
2611#
2612function del_group_freebsd #<group_name>
2613{
2614	typeset group=$1
2615
2616	pw groupdel -n $group > /dev/null 2>&1
2617	typeset -i ret=$?
2618	case $ret in
2619		# Group does not exist, or was deleted successfully.
2620		0|6|65) return 0 ;;
2621		# Name already exists as a group name
2622		9) log_must pw groupdel $group ;;
2623		*) return 1 ;;
2624	esac
2625
2626	return 0
2627}
2628
2629function add_user_illumos #<group_name> <user_name> <basedir>
2630{
2631	typeset group=$1
2632	typeset user=$2
2633	typeset basedir=$3
2634
2635	log_must useradd -g $group -d $basedir/$user -m $user
2636
2637	return 0
2638}
2639
2640function del_user_illumos #<user_name>
2641{
2642	typeset user=$1
2643
2644	if id $user > /dev/null 2>&1; then
2645		log_must_retry "currently used" 6 userdel $user
2646	fi
2647
2648	return 0
2649}
2650
2651function add_group_illumos #<group_name>
2652{
2653	typeset group=$1
2654
2655	typeset -i gid=100
2656	while true; do
2657		groupadd -g $gid $group > /dev/null 2>&1
2658		typeset -i ret=$?
2659		case $ret in
2660			0) return 0 ;;
2661			# The gid is not  unique
2662			4) ((gid += 1)) ;;
2663			*) return 1 ;;
2664		esac
2665	done
2666}
2667
2668function del_group_illumos #<group_name>
2669{
2670	typeset group=$1
2671
2672	groupmod -n $grp $grp > /dev/null 2>&1
2673	typeset -i ret=$?
2674	case $ret in
2675		# Group does not exist.
2676		6) return 0 ;;
2677		# Name already exists as a group name
2678		9) log_must groupdel $grp ;;
2679		*) return 1 ;;
2680	esac
2681}
2682
2683function add_user_linux #<group_name> <user_name> <basedir>
2684{
2685	typeset group=$1
2686	typeset user=$2
2687	typeset basedir=$3
2688
2689	log_must useradd -g $group -d $basedir/$user -m $user
2690
2691	# Add new users to the same group and the command line utils.
2692	# This allows them to be run out of the original users home
2693	# directory as long as it permissioned to be group readable.
2694	cmd_group=$(stat --format="%G" $(which zfs))
2695	log_must usermod -a -G $cmd_group $user
2696
2697	return 0
2698}
2699
2700function del_user_linux #<user_name>
2701{
2702	typeset user=$1
2703
2704	if id $user > /dev/null 2>&1; then
2705		log_must_retry "currently used" 6 userdel $user
2706	fi
2707
2708	return 0
2709}
2710
2711function add_group_linux #<group_name>
2712{
2713	typeset group=$1
2714
2715	# Assign 100 as the base gid, a larger value is selected for
2716	# Linux because for many distributions 1000 and under are reserved.
2717	while true; do
2718		groupadd $group > /dev/null 2>&1
2719		typeset -i ret=$?
2720		case $ret in
2721			0) return 0 ;;
2722			*) return 1 ;;
2723		esac
2724	done
2725}
2726
2727function del_group_linux #<group_name>
2728{
2729	typeset group=$1
2730
2731	getent group $group > /dev/null 2>&1
2732	typeset -i ret=$?
2733	case $ret in
2734		# Group does not exist.
2735		2) return 0 ;;
2736		# Name already exists as a group name
2737		0) log_must groupdel $group ;;
2738		*) return 1 ;;
2739	esac
2740
2741	return 0
2742}
2743
2744#
2745# Add specified user to specified group
2746#
2747# $1 group name
2748# $2 user name
2749# $3 base of the homedir (optional)
2750#
2751function add_user #<group_name> <user_name> <basedir>
2752{
2753	typeset group=$1
2754	typeset user=$2
2755	typeset basedir=${3:-"/var/tmp"}
2756
2757	if ((${#group} == 0 || ${#user} == 0)); then
2758		log_fail "group name or user name are not defined."
2759	fi
2760
2761	case $(uname) in
2762	FreeBSD)
2763		add_user_freebsd "$group" "$user" "$basedir"
2764		;;
2765	Linux)
2766		add_user_linux "$group" "$user" "$basedir"
2767		;;
2768	*)
2769		add_user_illumos "$group" "$user" "$basedir"
2770		;;
2771	esac
2772
2773	return 0
2774}
2775
2776#
2777# Delete the specified user.
2778#
2779# $1 login name
2780# $2 base of the homedir (optional)
2781#
2782function del_user #<logname> <basedir>
2783{
2784	typeset user=$1
2785	typeset basedir=${2:-"/var/tmp"}
2786
2787	if ((${#user} == 0)); then
2788		log_fail "login name is necessary."
2789	fi
2790
2791	case $(uname) in
2792	FreeBSD)
2793		del_user_freebsd "$user"
2794		;;
2795	Linux)
2796		del_user_linux "$user"
2797		;;
2798	*)
2799		del_user_illumos "$user"
2800		;;
2801	esac
2802
2803	[[ -d $basedir/$user ]] && rm -fr $basedir/$user
2804
2805	return 0
2806}
2807
2808#
2809# Select valid gid and create specified group.
2810#
2811# $1 group name
2812#
2813function add_group #<group_name>
2814{
2815	typeset group=$1
2816
2817	if ((${#group} == 0)); then
2818		log_fail "group name is necessary."
2819	fi
2820
2821	case $(uname) in
2822	FreeBSD)
2823		add_group_freebsd "$group"
2824		;;
2825	Linux)
2826		add_group_linux "$group"
2827		;;
2828	*)
2829		add_group_illumos "$group"
2830		;;
2831	esac
2832
2833	return 0
2834}
2835
2836#
2837# Delete the specified group.
2838#
2839# $1 group name
2840#
2841function del_group #<group_name>
2842{
2843	typeset group=$1
2844
2845	if ((${#group} == 0)); then
2846		log_fail "group name is necessary."
2847	fi
2848
2849	case $(uname) in
2850	FreeBSD)
2851		del_group_freebsd "$group"
2852		;;
2853	Linux)
2854		del_group_linux "$group"
2855		;;
2856	*)
2857		del_group_illumos "$group"
2858		;;
2859	esac
2860
2861	return 0
2862}
2863
2864#
2865# This function will return true if it's safe to destroy the pool passed
2866# as argument 1. It checks for pools based on zvols and files, and also
2867# files contained in a pool that may have a different mountpoint.
2868#
2869function safe_to_destroy_pool { # $1 the pool name
2870
2871	typeset pool=""
2872	typeset DONT_DESTROY=""
2873
2874	# We check that by deleting the $1 pool, we're not
2875	# going to pull the rug out from other pools. Do this
2876	# by looking at all other pools, ensuring that they
2877	# aren't built from files or zvols contained in this pool.
2878
2879	for pool in $(zpool list -H -o name)
2880	do
2881		ALTMOUNTPOOL=""
2882
2883		# this is a list of the top-level directories in each of the
2884		# files that make up the path to the files the pool is based on
2885		FILEPOOL=$(zpool status -v $pool | grep /$1/ | \
2886			awk '{print $1}')
2887
2888		# this is a list of the zvols that make up the pool
2889		ZVOLPOOL=$(zpool status -v $pool | grep "$ZVOL_DEVDIR/$1$" \
2890		    | awk '{print $1}')
2891
2892		# also want to determine if it's a file-based pool using an
2893		# alternate mountpoint...
2894		POOL_FILE_DIRS=$(zpool status -v $pool | \
2895					grep / | awk '{print $1}' | \
2896					awk -F/ '{print $2}' | grep -v "dev")
2897
2898		for pooldir in $POOL_FILE_DIRS
2899		do
2900			OUTPUT=$(zfs list -H -r -o mountpoint $1 | \
2901					grep "${pooldir}$" | awk '{print $1}')
2902
2903			ALTMOUNTPOOL="${ALTMOUNTPOOL}${OUTPUT}"
2904		done
2905
2906
2907		if [ ! -z "$ZVOLPOOL" ]
2908		then
2909			DONT_DESTROY="true"
2910			log_note "Pool $pool is built from $ZVOLPOOL on $1"
2911		fi
2912
2913		if [ ! -z "$FILEPOOL" ]
2914		then
2915			DONT_DESTROY="true"
2916			log_note "Pool $pool is built from $FILEPOOL on $1"
2917		fi
2918
2919		if [ ! -z "$ALTMOUNTPOOL" ]
2920		then
2921			DONT_DESTROY="true"
2922			log_note "Pool $pool is built from $ALTMOUNTPOOL on $1"
2923		fi
2924	done
2925
2926	if [ -z "${DONT_DESTROY}" ]
2927	then
2928		return 0
2929	else
2930		log_note "Warning: it is not safe to destroy $1!"
2931		return 1
2932	fi
2933}
2934
2935#
2936# Verify zfs operation with -p option work as expected
2937# $1 operation, value could be create, clone or rename
2938# $2 dataset type, value could be fs or vol
2939# $3 dataset name
2940# $4 new dataset name
2941#
2942function verify_opt_p_ops
2943{
2944	typeset ops=$1
2945	typeset datatype=$2
2946	typeset dataset=$3
2947	typeset newdataset=$4
2948
2949	if [[ $datatype != "fs" && $datatype != "vol" ]]; then
2950		log_fail "$datatype is not supported."
2951	fi
2952
2953	# check parameters accordingly
2954	case $ops in
2955		create)
2956			newdataset=$dataset
2957			dataset=""
2958			if [[ $datatype == "vol" ]]; then
2959				ops="create -V $VOLSIZE"
2960			fi
2961			;;
2962		clone)
2963			if [[ -z $newdataset ]]; then
2964				log_fail "newdataset should not be empty" \
2965					"when ops is $ops."
2966			fi
2967			log_must datasetexists $dataset
2968			log_must snapexists $dataset
2969			;;
2970		rename)
2971			if [[ -z $newdataset ]]; then
2972				log_fail "newdataset should not be empty" \
2973					"when ops is $ops."
2974			fi
2975			log_must datasetexists $dataset
2976			;;
2977		*)
2978			log_fail "$ops is not supported."
2979			;;
2980	esac
2981
2982	# make sure the upper level filesystem does not exist
2983	destroy_dataset "${newdataset%/*}" "-rRf"
2984
2985	# without -p option, operation will fail
2986	log_mustnot zfs $ops $dataset $newdataset
2987	log_mustnot datasetexists $newdataset ${newdataset%/*}
2988
2989	# with -p option, operation should succeed
2990	log_must zfs $ops -p $dataset $newdataset
2991	block_device_wait
2992
2993	if ! datasetexists $newdataset ; then
2994		log_fail "-p option does not work for $ops"
2995	fi
2996
2997	# when $ops is create or clone, redo the operation still return zero
2998	if [[ $ops != "rename" ]]; then
2999		log_must zfs $ops -p $dataset $newdataset
3000	fi
3001
3002	return 0
3003}
3004
3005#
3006# Get configuration of pool
3007# $1 pool name
3008# $2 config name
3009#
3010function get_config
3011{
3012	typeset pool=$1
3013	typeset config=$2
3014	typeset alt_root
3015
3016	if ! poolexists "$pool" ; then
3017		return 1
3018	fi
3019	alt_root=$(zpool list -H $pool | awk '{print $NF}')
3020	if [[ $alt_root == "-" ]]; then
3021		value=$(zdb -C $pool | grep "$config:" | awk -F: \
3022		    '{print $2}')
3023	else
3024		value=$(zdb -e $pool | grep "$config:" | awk -F: \
3025		    '{print $2}')
3026	fi
3027	if [[ -n $value ]] ; then
3028		value=${value#'}
3029		value=${value%'}
3030	fi
3031	echo $value
3032
3033	return 0
3034}
3035
3036#
3037# Privated function. Random select one of items from arguments.
3038#
3039# $1 count
3040# $2-n string
3041#
3042function _random_get
3043{
3044	typeset cnt=$1
3045	shift
3046
3047	typeset str="$@"
3048	typeset -i ind
3049	((ind = RANDOM % cnt + 1))
3050
3051	typeset ret=$(echo "$str" | cut -f $ind -d ' ')
3052	echo $ret
3053}
3054
3055#
3056# Random select one of item from arguments which include NONE string
3057#
3058function random_get_with_non
3059{
3060	typeset -i cnt=$#
3061	((cnt =+ 1))
3062
3063	_random_get "$cnt" "$@"
3064}
3065
3066#
3067# Random select one of item from arguments which doesn't include NONE string
3068#
3069function random_get
3070{
3071	_random_get "$#" "$@"
3072}
3073
3074#
3075# Detect if the current system support slog
3076#
3077function verify_slog_support
3078{
3079	typeset dir=$TEST_BASE_DIR/disk.$$
3080	typeset pool=foo.$$
3081	typeset vdev=$dir/a
3082	typeset sdev=$dir/b
3083
3084	mkdir -p $dir
3085	mkfile $MINVDEVSIZE $vdev $sdev
3086
3087	typeset -i ret=0
3088	if ! zpool create -n $pool $vdev log $sdev > /dev/null 2>&1; then
3089		ret=1
3090	fi
3091	rm -r $dir
3092
3093	return $ret
3094}
3095
3096#
3097# The function will generate a dataset name with specific length
3098# $1, the length of the name
3099# $2, the base string to construct the name
3100#
3101function gen_dataset_name
3102{
3103	typeset -i len=$1
3104	typeset basestr="$2"
3105	typeset -i baselen=${#basestr}
3106	typeset -i iter=0
3107	typeset l_name=""
3108
3109	if ((len % baselen == 0)); then
3110		((iter = len / baselen))
3111	else
3112		((iter = len / baselen + 1))
3113	fi
3114	while ((iter > 0)); do
3115		l_name="${l_name}$basestr"
3116
3117		((iter -= 1))
3118	done
3119
3120	echo $l_name
3121}
3122
3123#
3124# Get cksum tuple of dataset
3125# $1 dataset name
3126#
3127# sample zdb output:
3128# Dataset data/test [ZPL], ID 355, cr_txg 2413856, 31.0K, 7 objects, rootbp
3129# DVA[0]=<0:803046400:200> DVA[1]=<0:81199000:200> [L0 DMU objset] fletcher4
3130# lzjb LE contiguous unique double size=800L/200P birth=2413856L/2413856P
3131# fill=7 cksum=11ce125712:643a9c18ee2:125e25238fca0:254a3f74b59744
3132function datasetcksum
3133{
3134	typeset cksum
3135	sync
3136	cksum=$(zdb -vvv $1 | grep "^Dataset $1 \[" | grep "cksum" \
3137		| awk -F= '{print $7}')
3138	echo $cksum
3139}
3140
3141#
3142# Get cksum of file
3143# #1 file path
3144#
3145function checksum
3146{
3147	typeset cksum
3148	cksum=$(cksum $1 | awk '{print $1}')
3149	echo $cksum
3150}
3151
3152#
3153# Get the given disk/slice state from the specific field of the pool
3154#
3155function get_device_state #pool disk field("", "spares","logs")
3156{
3157	typeset pool=$1
3158	typeset disk=${2#$DEV_DSKDIR/}
3159	typeset field=${3:-$pool}
3160
3161	state=$(zpool status -v "$pool" 2>/dev/null | \
3162		nawk -v device=$disk -v pool=$pool -v field=$field \
3163		'BEGIN {startconfig=0; startfield=0; }
3164		/config:/ {startconfig=1}
3165		(startconfig==1) && ($1==field) {startfield=1; next;}
3166		(startfield==1) && ($1==device) {print $2; exit;}
3167		(startfield==1) &&
3168		($1==field || $1 ~ "^spares$" || $1 ~ "^logs$") {startfield=0}')
3169	echo $state
3170}
3171
3172
3173#
3174# print the given directory filesystem type
3175#
3176# $1 directory name
3177#
3178function get_fstype
3179{
3180	typeset dir=$1
3181
3182	if [[ -z $dir ]]; then
3183		log_fail "Usage: get_fstype <directory>"
3184	fi
3185
3186	#
3187	#  $ df -n /
3188	#  /		  : ufs
3189	#
3190	df -n $dir | awk '{print $3}'
3191}
3192
3193#
3194# Given a disk, label it to VTOC regardless what label was on the disk
3195# $1 disk
3196#
3197function labelvtoc
3198{
3199	typeset disk=$1
3200	if [[ -z $disk ]]; then
3201		log_fail "The disk name is unspecified."
3202	fi
3203	typeset label_file=/var/tmp/labelvtoc.$$
3204	typeset arch=$(uname -p)
3205
3206	if is_linux || is_freebsd; then
3207		log_note "Currently unsupported by the test framework"
3208		return 1
3209	fi
3210
3211	if [[ $arch == "i386" ]]; then
3212		echo "label" > $label_file
3213		echo "0" >> $label_file
3214		echo "" >> $label_file
3215		echo "q" >> $label_file
3216		echo "q" >> $label_file
3217
3218		fdisk -B $disk >/dev/null 2>&1
3219		# wait a while for fdisk finishes
3220		sleep 60
3221	elif [[ $arch == "sparc" ]]; then
3222		echo "label" > $label_file
3223		echo "0" >> $label_file
3224		echo "" >> $label_file
3225		echo "" >> $label_file
3226		echo "" >> $label_file
3227		echo "q" >> $label_file
3228	else
3229		log_fail "unknown arch type"
3230	fi
3231
3232	format -e -s -d $disk -f $label_file
3233	typeset -i ret_val=$?
3234	rm -f $label_file
3235	#
3236	# wait the format to finish
3237	#
3238	sleep 60
3239	if ((ret_val != 0)); then
3240		log_fail "unable to label $disk as VTOC."
3241	fi
3242
3243	return 0
3244}
3245
3246#
3247# check if the system was installed as zfsroot or not
3248# return: 0 if zfsroot, non-zero if not
3249#
3250function is_zfsroot
3251{
3252	df -n / | grep zfs > /dev/null 2>&1
3253	return $?
3254}
3255
3256#
3257# get the root filesystem name if it's zfsroot system.
3258#
3259# return: root filesystem name
3260function get_rootfs
3261{
3262	typeset rootfs=""
3263
3264	if is_freebsd; then
3265		rootfs=$(mount -p | awk '$2 == "/" && $3 == "zfs" {print $1}')
3266	elif ! is_linux; then
3267		rootfs=$(awk '{if ($2 == "/" && $3 == "zfs") print $1}' \
3268			/etc/mnttab)
3269	fi
3270	if [[ -z "$rootfs" ]]; then
3271		log_fail "Can not get rootfs"
3272	fi
3273	zfs list $rootfs > /dev/null 2>&1
3274	if (($? == 0)); then
3275		echo $rootfs
3276	else
3277		log_fail "This is not a zfsroot system."
3278	fi
3279}
3280
3281#
3282# get the rootfs's pool name
3283# return:
3284#       rootpool name
3285#
3286function get_rootpool
3287{
3288	typeset rootfs=""
3289	typeset rootpool=""
3290
3291	if is_freebsd; then
3292		rootfs=$(mount -p | awk '$2 == "/" && $3 == "zfs" {print $1}')
3293	elif ! is_linux; then
3294		rootfs=$(awk '{if ($2 == "/" && $3 =="zfs") print $1}' \
3295			 /etc/mnttab)
3296	fi
3297	if [[ -z "$rootfs" ]]; then
3298		log_fail "Can not get rootpool"
3299	fi
3300	zfs list $rootfs > /dev/null 2>&1
3301	if (($? == 0)); then
3302		echo ${rootfs%%/*}
3303	else
3304		log_fail "This is not a zfsroot system."
3305	fi
3306}
3307
3308#
3309# Get the word numbers from a string separated by white space
3310#
3311function get_word_count
3312{
3313	echo $1 | wc -w
3314}
3315
3316#
3317# To verify if the require numbers of disks is given
3318#
3319function verify_disk_count
3320{
3321	typeset -i min=${2:-1}
3322
3323	typeset -i count=$(get_word_count "$1")
3324
3325	if ((count < min)); then
3326		log_untested "A minimum of $min disks is required to run." \
3327			" You specified $count disk(s)"
3328	fi
3329}
3330
3331function ds_is_volume
3332{
3333	typeset type=$(get_prop type $1)
3334	[[ $type = "volume" ]] && return 0
3335	return 1
3336}
3337
3338function ds_is_filesystem
3339{
3340	typeset type=$(get_prop type $1)
3341	[[ $type = "filesystem" ]] && return 0
3342	return 1
3343}
3344
3345function ds_is_snapshot
3346{
3347	typeset type=$(get_prop type $1)
3348	[[ $type = "snapshot" ]] && return 0
3349	return 1
3350}
3351
3352#
3353# Check if Trusted Extensions are installed and enabled
3354#
3355function is_te_enabled
3356{
3357	svcs -H -o state labeld 2>/dev/null | grep "enabled"
3358	if (($? != 0)); then
3359		return 1
3360	else
3361		return 0
3362	fi
3363}
3364
3365# Return the number of CPUs (cross-platform)
3366function get_num_cpus
3367{
3368	if is_linux ; then
3369		grep -c '^processor' /proc/cpuinfo
3370	elif is_freebsd; then
3371		sysctl -n kern.smp.cpus
3372	else
3373		psrinfo | wc -l
3374	fi
3375}
3376
3377# Utility function to determine if a system has multiple cpus.
3378function is_mp
3379{
3380	if is_linux; then
3381		(($(nproc) > 1))
3382	elif is_freebsd; then
3383		sysctl -n kern.smp.cpus
3384	else
3385		(($(psrinfo | wc -l) > 1))
3386	fi
3387
3388	return $?
3389}
3390
3391function get_cpu_freq
3392{
3393	if is_linux; then
3394		lscpu | awk '/CPU MHz/ { print $3 }'
3395	elif is_freebsd; then
3396		sysctl -n hw.clockrate
3397	else
3398		psrinfo -v 0 | awk '/processor operates at/ {print $6}'
3399	fi
3400}
3401
3402# Run the given command as the user provided.
3403function user_run
3404{
3405	typeset user=$1
3406	shift
3407
3408	log_note "user: $user"
3409	log_note "cmd: $*"
3410
3411	typeset out=$TEST_BASE_DIR/out
3412	typeset err=$TEST_BASE_DIR/err
3413
3414	sudo -Eu $user env PATH="$PATH" ksh <<<"$*" >$out 2>$err
3415	typeset res=$?
3416	log_note "out: $(<$out)"
3417	log_note "err: $(<$err)"
3418	return $res
3419}
3420
3421#
3422# Check if the pool contains the specified vdevs
3423#
3424# $1 pool
3425# $2..n <vdev> ...
3426#
3427# Return 0 if the vdevs are contained in the pool, 1 if any of the specified
3428# vdevs is not in the pool, and 2 if pool name is missing.
3429#
3430function vdevs_in_pool
3431{
3432	typeset pool=$1
3433	typeset vdev
3434
3435	if [[ -z $pool ]]; then
3436		log_note "Missing pool name."
3437		return 2
3438	fi
3439
3440	shift
3441
3442	# We could use 'zpool list' to only get the vdevs of the pool but we
3443	# can't reference a mirror/raidz vdev using its ID (i.e mirror-0),
3444	# therefore we use the 'zpool status' output.
3445	typeset tmpfile=$(mktemp)
3446	zpool status -v "$pool" | grep -A 1000 "config:" >$tmpfile
3447	for vdev in $@; do
3448		grep -w ${vdev##*/} $tmpfile >/dev/null 2>&1
3449		[[ $? -ne 0 ]] && return 1
3450	done
3451
3452	rm -f $tmpfile
3453
3454	return 0;
3455}
3456
3457function get_max
3458{
3459	typeset -l i max=$1
3460	shift
3461
3462	for i in "$@"; do
3463		max=$((max > i ? max : i))
3464	done
3465
3466	echo $max
3467}
3468
3469function get_min
3470{
3471	typeset -l i min=$1
3472	shift
3473
3474	for i in "$@"; do
3475		min=$((min < i ? min : i))
3476	done
3477
3478	echo $min
3479}
3480
3481# Write data that can be compressed into a directory
3482function write_compressible
3483{
3484	typeset dir=$1
3485	typeset megs=$2
3486	typeset nfiles=${3:-1}
3487	typeset bs=${4:-1024k}
3488	typeset fname=${5:-file}
3489
3490	[[ -d $dir ]] || log_fail "No directory: $dir"
3491
3492	# Under Linux fio is not currently used since its behavior can
3493	# differ significantly across versions.  This includes missing
3494	# command line options and cases where the --buffer_compress_*
3495	# options fail to behave as expected.
3496	if is_linux; then
3497		typeset file_bytes=$(to_bytes $megs)
3498		typeset bs_bytes=4096
3499		typeset blocks=$(($file_bytes / $bs_bytes))
3500
3501		for (( i = 0; i < $nfiles; i++ )); do
3502			truncate -s $file_bytes $dir/$fname.$i
3503
3504			# Write every third block to get 66% compression.
3505			for (( j = 0; j < $blocks; j += 3 )); do
3506				dd if=/dev/urandom of=$dir/$fname.$i \
3507				    seek=$j bs=$bs_bytes count=1 \
3508				    conv=notrunc >/dev/null 2>&1
3509			done
3510		done
3511	else
3512		log_must eval "fio \
3513		    --name=job \
3514		    --fallocate=0 \
3515		    --minimal \
3516		    --randrepeat=0 \
3517		    --buffer_compress_percentage=66 \
3518		    --buffer_compress_chunk=4096 \
3519		    --directory=$dir \
3520		    --numjobs=$nfiles \
3521		    --nrfiles=$nfiles \
3522		    --rw=write \
3523		    --bs=$bs \
3524		    --filesize=$megs \
3525		    --filename_format='$fname.\$jobnum' >/dev/null"
3526	fi
3527}
3528
3529function get_objnum
3530{
3531	typeset pathname=$1
3532	typeset objnum
3533
3534	[[ -e $pathname ]] || log_fail "No such file or directory: $pathname"
3535	if is_freebsd; then
3536		objnum=$(stat -f "%i" $pathname)
3537	else
3538		objnum=$(stat -c %i $pathname)
3539	fi
3540	echo $objnum
3541}
3542
3543#
3544# Sync data to the pool
3545#
3546# $1 pool name
3547# $2 boolean to force uberblock (and config including zpool cache file) update
3548#
3549function sync_pool #pool <force>
3550{
3551	typeset pool=${1:-$TESTPOOL}
3552	typeset force=${2:-false}
3553
3554	if [[ $force == true ]]; then
3555		log_must zpool sync -f $pool
3556	else
3557		log_must zpool sync $pool
3558	fi
3559
3560	return 0
3561}
3562
3563#
3564# Wait for zpool 'freeing' property drops to zero.
3565#
3566# $1 pool name
3567#
3568function wait_freeing #pool
3569{
3570	typeset pool=${1:-$TESTPOOL}
3571	while true; do
3572		[[ "0" == "$(zpool list -Ho freeing $pool)" ]] && break
3573		log_must sleep 1
3574	done
3575}
3576
3577#
3578# Wait for every device replace operation to complete
3579#
3580# $1 pool name
3581#
3582function wait_replacing #pool
3583{
3584	typeset pool=${1:-$TESTPOOL}
3585	while true; do
3586		[[ "" == "$(zpool status $pool |
3587		    awk '/replacing-[0-9]+/ {print $1}')" ]] && break
3588		log_must sleep 1
3589	done
3590}
3591
3592#
3593# Wait for a pool to be scrubbed
3594#
3595# $1 pool name
3596#
3597function wait_scrubbed
3598{
3599	typeset pool=${1:-$TESTPOOL}
3600	while ! is_pool_scrubbed $pool ; do
3601		sleep 1
3602	done
3603}
3604
3605# Backup the zed.rc in our test directory so that we can edit it for our test.
3606#
3607# Returns: Backup file name.  You will need to pass this to zed_rc_restore().
3608function zed_rc_backup
3609{
3610	zedrc_backup="$(mktemp)"
3611	cp $ZEDLET_DIR/zed.rc $zedrc_backup
3612	echo $zedrc_backup
3613}
3614
3615function zed_rc_restore
3616{
3617	mv $1 $ZEDLET_DIR/zed.rc
3618}
3619
3620#
3621# Setup custom environment for the ZED.
3622#
3623# $@ Optional list of zedlets to run under zed.
3624function zed_setup
3625{
3626	if ! is_linux; then
3627		log_unsupported "No zed on $(uname)"
3628	fi
3629
3630	if [[ ! -d $ZEDLET_DIR ]]; then
3631		log_must mkdir $ZEDLET_DIR
3632	fi
3633
3634	if [[ ! -e $VDEVID_CONF ]]; then
3635		log_must touch $VDEVID_CONF
3636	fi
3637
3638	if [[ -e $VDEVID_CONF_ETC ]]; then
3639		log_fail "Must not have $VDEVID_CONF_ETC file present on system"
3640	fi
3641	EXTRA_ZEDLETS=$@
3642
3643	# Create a symlink for /etc/zfs/vdev_id.conf file.
3644	log_must ln -s $VDEVID_CONF $VDEVID_CONF_ETC
3645
3646	# Setup minimal ZED configuration.  Individual test cases should
3647	# add additional ZEDLETs as needed for their specific test.
3648	log_must cp ${ZEDLET_ETC_DIR}/zed.rc $ZEDLET_DIR
3649	log_must cp ${ZEDLET_ETC_DIR}/zed-functions.sh $ZEDLET_DIR
3650
3651	# Scripts must only be user writable.
3652	if [[ -n "$EXTRA_ZEDLETS" ]] ; then
3653		saved_umask=$(umask)
3654		log_must umask 0022
3655		for i in $EXTRA_ZEDLETS ; do
3656			log_must cp ${ZEDLET_LIBEXEC_DIR}/$i $ZEDLET_DIR
3657		done
3658		log_must umask $saved_umask
3659	fi
3660
3661	# Customize the zed.rc file to enable the full debug log.
3662	log_must sed -i '/\#ZED_DEBUG_LOG=.*/d' $ZEDLET_DIR/zed.rc
3663	echo "ZED_DEBUG_LOG=$ZED_DEBUG_LOG" >>$ZEDLET_DIR/zed.rc
3664
3665}
3666
3667#
3668# Cleanup custom ZED environment.
3669#
3670# $@ Optional list of zedlets to remove from our test zed.d directory.
3671function zed_cleanup
3672{
3673	if ! is_linux; then
3674		return
3675	fi
3676	EXTRA_ZEDLETS=$@
3677
3678	log_must rm -f ${ZEDLET_DIR}/zed.rc
3679	log_must rm -f ${ZEDLET_DIR}/zed-functions.sh
3680	log_must rm -f ${ZEDLET_DIR}/all-syslog.sh
3681	log_must rm -f ${ZEDLET_DIR}/all-debug.sh
3682	log_must rm -f ${ZEDLET_DIR}/state
3683
3684	if [[ -n "$EXTRA_ZEDLETS" ]] ; then
3685		for i in $EXTRA_ZEDLETS ; do
3686			log_must rm -f ${ZEDLET_DIR}/$i
3687		done
3688	fi
3689	log_must rm -f $ZED_LOG
3690	log_must rm -f $ZED_DEBUG_LOG
3691	log_must rm -f $VDEVID_CONF_ETC
3692	log_must rm -f $VDEVID_CONF
3693	rmdir $ZEDLET_DIR
3694}
3695
3696#
3697# Check if ZED is currently running, if not start ZED.
3698#
3699function zed_start
3700{
3701	if ! is_linux; then
3702		return
3703	fi
3704
3705	# ZEDLET_DIR=/var/tmp/zed
3706	if [[ ! -d $ZEDLET_DIR ]]; then
3707		log_must mkdir $ZEDLET_DIR
3708	fi
3709
3710	# Verify the ZED is not already running.
3711	pgrep -x zed > /dev/null
3712	if (($? == 0)); then
3713		log_note "ZED already running"
3714	else
3715		log_note "Starting ZED"
3716		# run ZED in the background and redirect foreground logging
3717		# output to $ZED_LOG.
3718		log_must truncate -s 0 $ZED_DEBUG_LOG
3719		log_must eval "zed -vF -d $ZEDLET_DIR -P $PATH" \
3720		    "-s $ZEDLET_DIR/state -j 1 2>$ZED_LOG &"
3721	fi
3722
3723	return 0
3724}
3725
3726#
3727# Kill ZED process
3728#
3729function zed_stop
3730{
3731	if ! is_linux; then
3732		return
3733	fi
3734
3735	log_note "Stopping ZED"
3736	while true; do
3737		zedpids="$(pgrep -x zed)"
3738		[ "$?" -ne 0 ] && break
3739
3740		log_must kill $zedpids
3741		sleep 1
3742	done
3743	return 0
3744}
3745
3746#
3747# Drain all zevents
3748#
3749function zed_events_drain
3750{
3751	while [ $(zpool events -H | wc -l) -ne 0 ]; do
3752		sleep 1
3753		zpool events -c >/dev/null
3754	done
3755}
3756
3757# Set a variable in zed.rc to something, un-commenting it in the process.
3758#
3759# $1 variable
3760# $2 value
3761function zed_rc_set
3762{
3763	var="$1"
3764	val="$2"
3765	# Remove the line
3766	cmd="'/$var/d'"
3767	eval sed -i $cmd $ZEDLET_DIR/zed.rc
3768
3769	# Add it at the end
3770	echo "$var=$val" >> $ZEDLET_DIR/zed.rc
3771}
3772
3773
3774#
3775# Check is provided device is being active used as a swap device.
3776#
3777function is_swap_inuse
3778{
3779	typeset device=$1
3780
3781	if [[ -z $device ]] ; then
3782		log_note "No device specified."
3783		return 1
3784	fi
3785
3786	if is_linux; then
3787		swapon -s | grep -w $(readlink -f $device) > /dev/null 2>&1
3788	elif is_freebsd; then
3789		swapctl -l | grep -w $device
3790	else
3791		swap -l | grep -w $device > /dev/null 2>&1
3792	fi
3793
3794	return $?
3795}
3796
3797#
3798# Setup a swap device using the provided device.
3799#
3800function swap_setup
3801{
3802	typeset swapdev=$1
3803
3804	if is_linux; then
3805		log_must eval "mkswap $swapdev > /dev/null 2>&1"
3806		log_must swapon $swapdev
3807	elif is_freebsd; then
3808		log_must swapctl -a $swapdev
3809	else
3810	        log_must swap -a $swapdev
3811	fi
3812
3813	return 0
3814}
3815
3816#
3817# Cleanup a swap device on the provided device.
3818#
3819function swap_cleanup
3820{
3821	typeset swapdev=$1
3822
3823	if is_swap_inuse $swapdev; then
3824		if is_linux; then
3825			log_must swapoff $swapdev
3826		elif is_freebsd; then
3827			log_must swapoff $swapdev
3828		else
3829			log_must swap -d $swapdev
3830		fi
3831	fi
3832
3833	return 0
3834}
3835
3836#
3837# Set a global system tunable (64-bit value)
3838#
3839# $1 tunable name (use a NAME defined in tunables.cfg)
3840# $2 tunable values
3841#
3842function set_tunable64
3843{
3844	set_tunable_impl "$1" "$2" Z
3845}
3846
3847#
3848# Set a global system tunable (32-bit value)
3849#
3850# $1 tunable name (use a NAME defined in tunables.cfg)
3851# $2 tunable values
3852#
3853function set_tunable32
3854{
3855	set_tunable_impl "$1" "$2" W
3856}
3857
3858function set_tunable_impl
3859{
3860	typeset name="$1"
3861	typeset value="$2"
3862	typeset mdb_cmd="$3"
3863	typeset module="${4:-zfs}"
3864
3865	eval "typeset tunable=\$$name"
3866	case "$tunable" in
3867	UNSUPPORTED)
3868		log_unsupported "Tunable '$name' is unsupported on $(uname)"
3869		;;
3870	"")
3871		log_fail "Tunable '$name' must be added to tunables.cfg"
3872		;;
3873	*)
3874		;;
3875	esac
3876
3877	[[ -z "$value" ]] && return 1
3878	[[ -z "$mdb_cmd" ]] && return 1
3879
3880	case "$(uname)" in
3881	Linux)
3882		typeset zfs_tunables="/sys/module/$module/parameters"
3883		[[ -w "$zfs_tunables/$tunable" ]] || return 1
3884		cat >"$zfs_tunables/$tunable" <<<"$value"
3885		return $?
3886		;;
3887	FreeBSD)
3888		sysctl vfs.zfs.$tunable=$value
3889		return "$?"
3890		;;
3891	SunOS)
3892		[[ "$module" -eq "zfs" ]] || return 1
3893		echo "${tunable}/${mdb_cmd}0t${value}" | mdb -kw
3894		return $?
3895		;;
3896	esac
3897}
3898
3899#
3900# Get a global system tunable
3901#
3902# $1 tunable name (use a NAME defined in tunables.cfg)
3903#
3904function get_tunable
3905{
3906	get_tunable_impl "$1"
3907}
3908
3909function get_tunable_impl
3910{
3911	typeset name="$1"
3912	typeset module="${2:-zfs}"
3913
3914	eval "typeset tunable=\$$name"
3915	case "$tunable" in
3916	UNSUPPORTED)
3917		log_unsupported "Tunable '$name' is unsupported on $(uname)"
3918		;;
3919	"")
3920		log_fail "Tunable '$name' must be added to tunables.cfg"
3921		;;
3922	*)
3923		;;
3924	esac
3925
3926	case "$(uname)" in
3927	Linux)
3928		typeset zfs_tunables="/sys/module/$module/parameters"
3929		[[ -f "$zfs_tunables/$tunable" ]] || return 1
3930		cat $zfs_tunables/$tunable
3931		return $?
3932		;;
3933	FreeBSD)
3934		sysctl -n vfs.zfs.$tunable
3935		;;
3936	SunOS)
3937		[[ "$module" -eq "zfs" ]] || return 1
3938		;;
3939	esac
3940
3941	return 1
3942}
3943
3944#
3945# Prints the current time in seconds since UNIX Epoch.
3946#
3947function current_epoch
3948{
3949	printf '%(%s)T'
3950}
3951
3952#
3953# Get decimal value of global uint32_t variable using mdb.
3954#
3955function mdb_get_uint32
3956{
3957	typeset variable=$1
3958	typeset value
3959
3960	value=$(mdb -k -e "$variable/X | ::eval .=U")
3961	if [[ $? -ne 0 ]]; then
3962		log_fail "Failed to get value of '$variable' from mdb."
3963		return 1
3964	fi
3965
3966	echo $value
3967	return 0
3968}
3969
3970#
3971# Set global uint32_t variable to a decimal value using mdb.
3972#
3973function mdb_set_uint32
3974{
3975	typeset variable=$1
3976	typeset value=$2
3977
3978	mdb -kw -e "$variable/W 0t$value" > /dev/null
3979	if [[ $? -ne 0 ]]; then
3980		echo "Failed to set '$variable' to '$value' in mdb."
3981		return 1
3982	fi
3983
3984	return 0
3985}
3986
3987#
3988# Set global scalar integer variable to a hex value using mdb.
3989# Note: Target should have CTF data loaded.
3990#
3991function mdb_ctf_set_int
3992{
3993	typeset variable=$1
3994	typeset value=$2
3995
3996	mdb -kw -e "$variable/z $value" > /dev/null
3997	if [[ $? -ne 0 ]]; then
3998		echo "Failed to set '$variable' to '$value' in mdb."
3999		return 1
4000	fi
4001
4002	return 0
4003}
4004
4005#
4006# Compute MD5 digest for given file or stdin if no file given.
4007# Note: file path must not contain spaces
4008#
4009function md5digest
4010{
4011	typeset file=$1
4012
4013	case $(uname) in
4014	FreeBSD)
4015		md5 -q $file
4016		;;
4017	*)
4018		md5sum -b $file | awk '{ print $1 }'
4019		;;
4020	esac
4021}
4022
4023#
4024# Compute SHA256 digest for given file or stdin if no file given.
4025# Note: file path must not contain spaces
4026#
4027function sha256digest
4028{
4029	typeset file=$1
4030
4031	case $(uname) in
4032	FreeBSD)
4033		sha256 -q $file
4034		;;
4035	*)
4036		sha256sum -b $file | awk '{ print $1 }'
4037		;;
4038	esac
4039}
4040
4041function new_fs #<args>
4042{
4043	case $(uname) in
4044	FreeBSD)
4045		newfs "$@"
4046		;;
4047	*)
4048		echo y | newfs -v "$@"
4049		;;
4050	esac
4051}
4052
4053function stat_size #<path>
4054{
4055	typeset path=$1
4056
4057	case $(uname) in
4058	FreeBSD)
4059		stat -f %z "$path"
4060		;;
4061	*)
4062		stat -c %s "$path"
4063		;;
4064	esac
4065}
4066
4067function stat_ctime #<path>
4068{
4069	typeset path=$1
4070
4071	case $(uname) in
4072	FreeBSD)
4073		stat -f %c "$path"
4074		;;
4075	*)
4076		stat -c %Z "$path"
4077		;;
4078	esac
4079}
4080
4081function stat_crtime #<path>
4082{
4083	typeset path=$1
4084
4085	case $(uname) in
4086	FreeBSD)
4087		stat -f %B "$path"
4088		;;
4089	*)
4090		stat -c %W "$path"
4091		;;
4092	esac
4093}
4094
4095# Run a command as if it was being run in a TTY.
4096#
4097# Usage:
4098#
4099#    faketty command
4100#
4101function faketty
4102{
4103    if is_freebsd; then
4104        script -q /dev/null env "$@"
4105    else
4106        script --return --quiet -c "$*" /dev/null
4107    fi
4108}
4109
4110#
4111# Produce a random permutation of the integers in a given range (inclusive).
4112#
4113function range_shuffle # begin end
4114{
4115	typeset -i begin=$1
4116	typeset -i end=$2
4117
4118	seq ${begin} ${end} | sort -R
4119}
4120
4121#
4122# Cross-platform xattr helpers
4123#
4124
4125function get_xattr # name path
4126{
4127	typeset name=$1
4128	typeset path=$2
4129
4130	case $(uname) in
4131	FreeBSD)
4132		getextattr -qq user "${name}" "${path}"
4133		;;
4134	*)
4135		attr -qg "${name}" "${path}"
4136		;;
4137	esac
4138}
4139
4140function set_xattr # name value path
4141{
4142	typeset name=$1
4143	typeset value=$2
4144	typeset path=$3
4145
4146	case $(uname) in
4147	FreeBSD)
4148		setextattr user "${name}" "${value}" "${path}"
4149		;;
4150	*)
4151		attr -qs "${name}" -V "${value}" "${path}"
4152		;;
4153	esac
4154}
4155
4156function set_xattr_stdin # name value
4157{
4158	typeset name=$1
4159	typeset path=$2
4160
4161	case $(uname) in
4162	FreeBSD)
4163		setextattr -i user "${name}" "${path}"
4164		;;
4165	*)
4166		attr -qs "${name}" "${path}"
4167		;;
4168	esac
4169}
4170
4171function rm_xattr # name path
4172{
4173	typeset name=$1
4174	typeset path=$2
4175
4176	case $(uname) in
4177	FreeBSD)
4178		rmextattr -q user "${name}" "${path}"
4179		;;
4180	*)
4181		attr -qr "${name}" "${path}"
4182		;;
4183	esac
4184}
4185
4186function ls_xattr # path
4187{
4188	typeset path=$1
4189
4190	case $(uname) in
4191	FreeBSD)
4192		lsextattr -qq user "${path}"
4193		;;
4194	*)
4195		attr -ql "${path}"
4196		;;
4197	esac
4198}
4199
4200function kstat # stat flags?
4201{
4202	typeset stat=$1
4203	typeset flags=${2-"-n"}
4204
4205	case $(uname) in
4206	FreeBSD)
4207		sysctl $flags kstat.zfs.misc.$stat
4208		;;
4209	Linux)
4210		typeset zfs_kstat="/proc/spl/kstat/zfs/$stat"
4211		[[ -f "$zfs_kstat" ]] || return 1
4212		cat $zfs_kstat
4213		;;
4214	*)
4215		false
4216		;;
4217	esac
4218}
4219
4220function get_arcstat # stat
4221{
4222	typeset stat=$1
4223
4224	case $(uname) in
4225	FreeBSD)
4226		kstat arcstats.$stat
4227		;;
4228	Linux)
4229		kstat arcstats | awk "/$stat/ { print \$3 }"
4230		;;
4231	*)
4232		false
4233		;;
4234	esac
4235}
4236
4237function punch_hole # offset length file
4238{
4239	typeset offset=$1
4240	typeset length=$2
4241	typeset file=$3
4242
4243	case $(uname) in
4244	FreeBSD)
4245		truncate -d -o $offset -l $length "$file"
4246		;;
4247	Linux)
4248		fallocate --punch-hole --offset $offset --length $length "$file"
4249		;;
4250	*)
4251		false
4252		;;
4253	esac
4254}
4255
4256function zero_range # offset length file
4257{
4258	typeset offset=$1
4259	typeset length=$2
4260	typeset file=$3
4261
4262	case "$UNAME" in
4263	Linux)
4264		fallocate --zero-range --offset $offset --length $length "$file"
4265		;;
4266	*)
4267		false
4268		;;
4269	esac
4270}
4271
4272#
4273# Wait for the specified arcstat to reach non-zero quiescence.
4274# If echo is 1 echo the value after reaching quiescence, otherwise
4275# if echo is 0 print the arcstat we are waiting on.
4276#
4277function arcstat_quiescence # stat echo
4278{
4279	typeset stat=$1
4280	typeset echo=$2
4281	typeset do_once=true
4282
4283	if [[ $echo -eq 0 ]]; then
4284		echo "Waiting for arcstat $1 quiescence."
4285	fi
4286
4287	while $do_once || [ $stat1 -ne $stat2 ] || [ $stat2 -eq 0 ]; do
4288		typeset stat1=$(get_arcstat $stat)
4289		sleep 0.5
4290		typeset stat2=$(get_arcstat $stat)
4291		do_once=false
4292	done
4293
4294	if [[ $echo -eq 1 ]]; then
4295		echo $stat2
4296	fi
4297}
4298
4299function arcstat_quiescence_noecho # stat
4300{
4301	typeset stat=$1
4302	arcstat_quiescence $stat 0
4303}
4304
4305function arcstat_quiescence_echo # stat
4306{
4307	typeset stat=$1
4308	arcstat_quiescence $stat 1
4309}
4310
4311#
4312# Given an array of pids, wait until all processes
4313# have completed and check their return status.
4314#
4315function wait_for_children #children
4316{
4317	rv=0
4318	children=("$@")
4319	for child in "${children[@]}"
4320	do
4321		child_exit=0
4322		wait ${child} || child_exit=$?
4323		if [ $child_exit -ne 0 ]; then
4324			echo "child ${child} failed with ${child_exit}"
4325			rv=1
4326		fi
4327	done
4328	return $rv
4329}
4330