zpool iostat -v 1
#mounting an encrypted filesystem
zfs mount -l -a
#adding slog to a pool
zpool add zfs log /dev/nvme0n1
#adding l2arc to a pool
zpool add zfs cache /dev/nvme0n2
#checking all zfs module parameters
modinfo zfs
#check atime enabled?
zfs get all |grep atime
#disable atime
zfs set atime=off zfs
zfs set atime=off zfs_sata
#check serial number of disk
[root@node-2 ~]# sginfo -s /dev/sda
Serial Number 'PDNLH0BRH9F2DJ'
#check physical and logical block size of disks
lsblk -o NAME,PHY-SEC,LOG-SEC,SIZE,TYPE,ROTA
#physical vs. logical block size
- physical sector size -> actual hard drive reads and writes
- logical sector size -> supported smallest hard drive reads and writes
basically ashift means the exponent, index or power to the base of 2 e.g. for a physical sector size of 512 == 2^9 -> ashift of 9, or 4k == 2^12 -> ashift 12
#Example for 4x2TB rotational disks
[root@node-2 ~]# lsblk -o NAME,PHY-SEC,LOG-SEC,SIZE,TYPE,ROTA |grep 'disk 1'
sdk 4096 512 1.8T disk 1
sdi 4096 512 1.8T disk 1
sdl 4096 512 1.8T disk 1
sdj 4096 512 1.8T disk 1
[root@node-2 ~]# ls -l /dev/disk/by-id/
total 0
lrwxrwxrwx. 1 root root 9 Nov 17 23:14 ata-ST2000LM015-2E8174_WDZ3WZFN -> ../../sdj
lrwxrwxrwx. 1 root root 9 Nov 17 23:15 ata-ST2000LM015-2E8174_WDZAAC2H -> ../../sdi
lrwxrwxrwx. 1 root root 9 Nov 17 23:15 ata-ST2000LX001-1RG174_WDZASXRK -> ../../sdk
lrwxrwxrwx. 1 root root 9 Nov 17 23:15 ata-ST2000LX001-1RG174_ZDZ4TJK2 -> ../../sdl
#creating the pool
zpool create zfs_sata -o ashift=12 mirror ata-ST2000LM015-2E8174_WDZAAC2H ata-ST2000LM015-2E8174_WDZ3WZFN mirror ata-ST2000LX001-1RG174_WDZASXRK ata-ST2000LX001-1RG174_ZDZ4TJK2
#creating two volumes with different record sizes
zfs create -o recordsize=16k zfs_sata/mfdatabase
zfs create -o recordsize=1024k zfs_sata/mfjournal
#some sequential testing
dd if=/dev/zero of=/zfs_sata/mfdatabase/tempfile bs=1M count=1024; sync
dd if=/dev/zero of=/zfs_sata/mfdatabase/tempfile2 bs=16k count=65536; sync
/sbin/sysctl -w vm.drop_caches=3
dd if=/zfs_sata/mfdatabase/tempfile of=/dev/null bs=1M count=1024
dd if=/zfs_sata/mfdatabase/tempfile2 of=/dev/null bs=16k count=65536
rm -rf /zfs_sata/mfdatabase/*
writes
16.5013 s, 65.1 MB/s (expected slower)
7.26762 s, 148 MB/s (expected faster)
reads
54.6486 s, 19.6 MB/s (expected slower)
59.2402 s, 18.1 MB/s (expected slower)
dd if=/dev/zero of=/zfs_sata/mfjournal/tempfile bs=1M count=1024; sync
dd if=/dev/zero of=/zfs_sata/mfjournal/tempfile2 bs=16k count=65536; sync
/sbin/sysctl -w vm.drop_caches=3
dd if=/zfs_sata/mfjournal/tempfile of=/dev/null bs=1M count=1024
dd if=/zfs_sata/mfjournal/tempfile2 of=/dev/null bs=16k count=65536
rm -rf /zfs_sata/mfjournal/*
writes
12.1631 s, 88.3 MB/s (expected faster)
8.75189 s, 123 MB/s (expected slower)
reads
43.0267 s, 25.0 MB/s (expected faster)
23.1101 s, 46.5 MB/s (expected faster)
#zfs parameters
yum install sysfsutils -y
systool -vm zfs
#or
ls -l /sys/module/zfs/parameters/#change zfs options for good
/etc/modprobe.d/zfs.conf
e.g. options zfs PARAMETER=VALUE
#change zfs option in flight
echo NEWVALUE >> /sys/module/zfs/parameters/PARAMETER
#Example 16GiB for zfs_arc_max
echo 17179869184>> /sys/module/zfs/parameters/zfs_arc_max
[root@node-2d current]# systool -vm zfs |grep zfs_arc_max
zfs_arc_max = "17179869184"
zfs_arc_max = "17179869184"
zfs_arc_max
https://forums.freebsd.org/threads/howto-tuning-l2arc-in-zfs.29907/ l2arc_write_max: 8388608 # Maximum number of bytes written to l2arc per feed l2arc_write_boost: 8388608 # Mostly only relevant at the first few hours after boot l2arc_headroom: 2 # Not sure l2arc_feed_secs: 1 # l2arc feeding period l2arc_feed_min_ms: 200 # minimum l2arc feeding period l2arc_noprefetch: 1 # control whether streaming data is cached or not l2arc_feed_again: 1 # control whether feed_min_ms is used or not l2arc_norw: 1 # no read and write at the same time
/etc/modprobe.d/zfs.conf #log options zfs zfs_txg_timeout=30 #cache options zfs zfs_arc_max=34359738368 options zfs l2arc_noprefetch=0 options zfs l2arc_write_max=1073741824 options zfs l2arc_write_boost=2147483648
1073741824zil_slog_limit
#for all SSD pool logbias could be changed e.g.
zfs set logbias=throughput zfs/mydata
#look at performance
zpool iostat -v 1
#resources I read
- http://open-zfs.org/wiki/Performance_tuning
- https://github.com/zfsonlinux/zfs/wiki/ZFS-on-Linux-Module-Parameters
- https://martin.heiland.io/2018/02/23/zfs-tuning
- https://www.svennd.be/tuning-of-zfs-module
- https://utcc.utoronto.ca/~cks/space/blog/solaris/ZFSWritesAndZIL
- http://www.nanowolk.nl/ext/2013_02_zfs_sequential_read_write_performance
- http://www.nanowolk.nl/ext/2013_02_zfs_random_iops_read_write_performance
- https://sites.google.com/site/ryanbabchishin/home/publications/changing-a-zvol-block-size-while-making-it-sparse-and-compressed
- https://utcc.utoronto.ca/~cks/space/blog/tech/AdvancedFormatDrives
- https://docs.oracle.com/cd/E23823_01/html/819-5461/gazss.html#indexterm-425
- https://zfs.datto.com/2017_slides/pinchuk.pdf
