diff --git a/Documentation/ABI/testing/configfs-usb-gadget-uac1_legacy b/Documentation/ABI/testing/configfs-usb-gadget-uac1_legacy index b2eaefd9bc498b..6a681d219f439b 100644 --- a/Documentation/ABI/testing/configfs-usb-gadget-uac1_legacy +++ b/Documentation/ABI/testing/configfs-usb-gadget-uac1_legacy @@ -5,8 +5,5 @@ Description: The attributes: audio_buf_size - audio buffer size - fn_cap - capture pcm device file name - fn_cntl - control device file name - fn_play - playback pcm device file name req_buf_size - ISO OUT endpoint request buffer size req_count - ISO OUT endpoint request count diff --git a/Documentation/admin-guide/device-mapper/dm-ima.rst b/Documentation/admin-guide/device-mapper/dm-ima.rst index a4aa50a828e005..2a3b50ffbee4e5 100644 --- a/Documentation/admin-guide/device-mapper/dm-ima.rst +++ b/Documentation/admin-guide/device-mapper/dm-ima.rst @@ -424,7 +424,8 @@ section above) has the following data format for 'integrity' target. target_attributes := "," "," "," "," "," [ ","] [ ","] "," - "," "," "," "," + "," "," "," "," + "," "," "," ";" target_name := "target_name=integrity" @@ -438,6 +439,7 @@ section above) has the following data format for 'integrity' target. block_size := "block_size=" recalculate := "recalculate=" allow_discards := "allow_discards=" + allow_discards_keyed := "allow_discards_keyed=" fix_padding := "fix_padding=" fix_hmac := "fix_hmac=" legacy_recalculate := "legacy_recalculate=" @@ -455,7 +457,8 @@ section above) has the following data format for 'integrity' target. dm_version=4.45.0; name=integrity1,uuid=,major=253,minor=1,minor_count=1,num_targets=1; target_index=0,target_begin=0,target_len=7856,target_name=integrity,target_version=1.10.0, - dev_name=253:0,start=0,tag_size=32,mode=J,recalculate=n,allow_discards=n,fix_padding=n, + dev_name=253:0,start=0,tag_size=32,mode=J,recalculate=n,allow_discards=n, + allow_discards_keyed=n,fix_padding=n, fix_hmac=n,legacy_recalculate=n,journal_sectors=88,interleave_sectors=32768,buffer_sectors=128; diff --git a/Documentation/admin-guide/device-mapper/dm-integrity.rst b/Documentation/admin-guide/device-mapper/dm-integrity.rst index c2e18ecc065c92..9c21301423c9e0 100644 --- a/Documentation/admin-guide/device-mapper/dm-integrity.rst +++ b/Documentation/admin-guide/device-mapper/dm-integrity.rst @@ -190,6 +190,19 @@ allow_discards Allow block discard requests (a.k.a. TRIM) for the integrity device. Discards are only allowed to devices using internal hash. + A discarded block is marked with a constant filler tag that anyone + with raw write access to the backing device can forge without the + key. Use allow_discards_keyed instead on new volumes. + +allow_discards_keyed + Like allow_discards, but marks a discarded block with a keyed + checksum of the sector number, HMAC_key(salt || sector), instead of + the constant filler tag, so it can't be forged without the + integrity key. + + Not compatible with volumes that already have discarded blocks + marked the old way; only use on a freshly formatted volume. + fix_padding Use a smaller padding of the tag area that is more space-efficient. If this option is not present, large padding is diff --git a/Documentation/hwmon/emc1403.rst b/Documentation/hwmon/emc1403.rst index 77060d5153238e..ce258f0fccc60b 100644 --- a/Documentation/hwmon/emc1403.rst +++ b/Documentation/hwmon/emc1403.rst @@ -71,10 +71,10 @@ and EMC14x8 support eight sensors (one internal, seven external). The chips implement three limits for each sensor: low (tempX_min), high (tempX_max) and critical (tempX_crit.) The chips also implement an -hysteresis mechanism which applies to all limits. The relative difference -is stored in a single register on the chip, which means that the relative -difference between the limit and its hysteresis is always the same for -all three limits. +hysteresis mechanism which applies to high and critical limits. The relative +difference is stored in a single register on the chip, which means that the +relative difference between the limit and its hysteresis is always the same +for high and critical limits. This implementation detail implies the following: diff --git a/Documentation/leds/leds-st1202.rst b/Documentation/leds/leds-st1202.rst index 1a09fbfcedcffe..a2353549469ee8 100644 --- a/Documentation/leds/leds-st1202.rst +++ b/Documentation/leds/leds-st1202.rst @@ -17,7 +17,7 @@ To be compatible with the hardware pattern format, maximum 8 tuples of brightness (PWM) and duration must be written to hw_pattern. - Min pattern duration: 22 ms -- Max pattern duration: 5660 ms +- Max pattern duration: 5610 ms The format of the hardware pattern values should be: "brightness duration brightness duration ..." diff --git a/Documentation/sunrpc/xdr/nfs4_1.x b/Documentation/sunrpc/xdr/nfs4_1.x index 5b45547b2ebc85..72d439b717356e 100644 --- a/Documentation/sunrpc/xdr/nfs4_1.x +++ b/Documentation/sunrpc/xdr/nfs4_1.x @@ -45,19 +45,162 @@ pragma header nfs4; /* * Basic typedefs for RFC 1832 data type definitions */ -typedef hyper int64_t; -typedef unsigned int uint32_t; +typedef int int32_t; +typedef unsigned int uint32_t; +typedef hyper int64_t; +typedef unsigned hyper uint64_t; + +const NFS4_VERIFIER_SIZE = 8; +const NFS4_FHSIZE = 128; + +enum nfsstat4 { + NFS4_OK = 0, /* everything is okay */ + NFS4ERR_PERM = 1, /* caller not privileged */ + NFS4ERR_NOENT = 2, /* no such file/directory */ + NFS4ERR_IO = 5, /* hard I/O error */ + NFS4ERR_NXIO = 6, /* no such device */ + NFS4ERR_ACCESS = 13, /* access denied */ + NFS4ERR_EXIST = 17, /* file already exists */ + NFS4ERR_XDEV = 18, /* different filesystems */ + + /* + * Please do not allocate value 19; it was used in NFSv3 + * and we do not want a value in NFSv3 to have a different + * meaning in NFSv4.x. + */ + + NFS4ERR_NOTDIR = 20, /* should be a directory */ + NFS4ERR_ISDIR = 21, /* should not be directory */ + NFS4ERR_INVAL = 22, /* invalid argument */ + NFS4ERR_FBIG = 27, /* file exceeds server max */ + NFS4ERR_NOSPC = 28, /* no space on filesystem */ + NFS4ERR_ROFS = 30, /* read-only filesystem */ + NFS4ERR_MLINK = 31, /* too many hard links */ + NFS4ERR_NAMETOOLONG = 63, /* name exceeds server max */ + NFS4ERR_NOTEMPTY = 66, /* directory not empty */ + NFS4ERR_DQUOT = 69, /* hard quota limit reached*/ + NFS4ERR_STALE = 70, /* file no longer exists */ + NFS4ERR_BADHANDLE = 10001,/* Illegal filehandle */ + NFS4ERR_BAD_COOKIE = 10003,/* READDIR cookie is stale */ + NFS4ERR_NOTSUPP = 10004,/* operation not supported */ + NFS4ERR_TOOSMALL = 10005,/* response limit exceeded */ + NFS4ERR_SERVERFAULT = 10006,/* undefined server error */ + NFS4ERR_BADTYPE = 10007,/* type invalid for CREATE */ + NFS4ERR_DELAY = 10008,/* file "busy" - retry */ + NFS4ERR_SAME = 10009,/* nverify says attrs same */ + NFS4ERR_DENIED = 10010,/* lock unavailable */ + NFS4ERR_EXPIRED = 10011,/* lock lease expired */ + NFS4ERR_LOCKED = 10012,/* I/O failed due to lock */ + NFS4ERR_GRACE = 10013,/* in grace period */ + NFS4ERR_FHEXPIRED = 10014,/* filehandle expired */ + NFS4ERR_SHARE_DENIED = 10015,/* share reserve denied */ + NFS4ERR_WRONGSEC = 10016,/* wrong security flavor */ + NFS4ERR_CLID_INUSE = 10017,/* clientid in use */ + + /* NFS4ERR_RESOURCE is not a valid error in NFSv4.1 */ + NFS4ERR_RESOURCE = 10018,/* resource exhaustion */ + + NFS4ERR_MOVED = 10019,/* filesystem relocated */ + NFS4ERR_NOFILEHANDLE = 10020,/* current FH is not set */ + NFS4ERR_MINOR_VERS_MISMATCH= 10021,/* minor vers not supp */ + NFS4ERR_STALE_CLIENTID = 10022,/* server has rebooted */ + NFS4ERR_STALE_STATEID = 10023,/* server has rebooted */ + NFS4ERR_OLD_STATEID = 10024,/* state is out of sync */ + NFS4ERR_BAD_STATEID = 10025,/* incorrect stateid */ + NFS4ERR_BAD_SEQID = 10026,/* request is out of seq. */ + NFS4ERR_NOT_SAME = 10027,/* verify - attrs not same */ + NFS4ERR_LOCK_RANGE = 10028,/* overlapping lock range */ + NFS4ERR_SYMLINK = 10029,/* should be file/directory*/ + NFS4ERR_RESTOREFH = 10030,/* no saved filehandle */ + NFS4ERR_LEASE_MOVED = 10031,/* some filesystem moved */ + NFS4ERR_ATTRNOTSUPP = 10032,/* recommended attr not sup*/ + NFS4ERR_NO_GRACE = 10033,/* reclaim outside of grace*/ + NFS4ERR_RECLAIM_BAD = 10034,/* reclaim error at server */ + NFS4ERR_RECLAIM_CONFLICT= 10035,/* conflict on reclaim */ + NFS4ERR_BADXDR = 10036,/* XDR decode failed */ + NFS4ERR_LOCKS_HELD = 10037,/* file locks held at CLOSE*/ + NFS4ERR_OPENMODE = 10038,/* conflict in OPEN and I/O*/ + NFS4ERR_BADOWNER = 10039,/* owner translation bad */ + NFS4ERR_BADCHAR = 10040,/* utf-8 char not supported*/ + NFS4ERR_BADNAME = 10041,/* name not supported */ + NFS4ERR_BAD_RANGE = 10042,/* lock range not supported*/ + NFS4ERR_LOCK_NOTSUPP = 10043,/* no atomic up/downgrade */ + NFS4ERR_OP_ILLEGAL = 10044,/* undefined operation */ + NFS4ERR_DEADLOCK = 10045,/* file locking deadlock */ + NFS4ERR_FILE_OPEN = 10046,/* open file blocks op. */ + NFS4ERR_ADMIN_REVOKED = 10047,/* lockowner state revoked */ + NFS4ERR_CB_PATH_DOWN = 10048,/* callback path down */ + + /* NFSv4.1 errors start here. */ + + NFS4ERR_BADIOMODE = 10049, + NFS4ERR_BADLAYOUT = 10050, + NFS4ERR_BAD_SESSION_DIGEST = 10051, + NFS4ERR_BADSESSION = 10052, + NFS4ERR_BADSLOT = 10053, + NFS4ERR_COMPLETE_ALREADY = 10054, + NFS4ERR_CONN_NOT_BOUND_TO_SESSION = 10055, + NFS4ERR_DELEG_ALREADY_WANTED = 10056, + NFS4ERR_BACK_CHAN_BUSY = 10057,/*backchan reqs outstanding*/ + NFS4ERR_LAYOUTTRYLATER = 10058, + NFS4ERR_LAYOUTUNAVAILABLE = 10059, + NFS4ERR_NOMATCHING_LAYOUT = 10060, + NFS4ERR_RECALLCONFLICT = 10061, + NFS4ERR_UNKNOWN_LAYOUTTYPE = 10062, + NFS4ERR_SEQ_MISORDERED = 10063,/* unexpected seq.ID in req*/ + NFS4ERR_SEQUENCE_POS = 10064,/* [CB_]SEQ. op not 1st op */ + NFS4ERR_REQ_TOO_BIG = 10065,/* request too big */ + NFS4ERR_REP_TOO_BIG = 10066,/* reply too big */ + NFS4ERR_REP_TOO_BIG_TO_CACHE =10067,/* rep. not all cached*/ + NFS4ERR_RETRY_UNCACHED_REP =10068,/* retry & rep. uncached*/ + NFS4ERR_UNSAFE_COMPOUND =10069,/* retry/recovery too hard */ + NFS4ERR_TOO_MANY_OPS = 10070,/*too many ops in [CB_]COMP*/ + NFS4ERR_OP_NOT_IN_SESSION =10071,/* op needs [CB_]SEQ. op */ + NFS4ERR_HASH_ALG_UNSUPP = 10072, /* hash alg. not supp. */ + /* Error 10073 is unused. */ + NFS4ERR_CLIENTID_BUSY = 10074,/* clientid has state */ + NFS4ERR_PNFS_IO_HOLE = 10075,/* IO to _SPARSE file hole */ + NFS4ERR_SEQ_FALSE_RETRY= 10076,/* Retry != original req. */ + NFS4ERR_BAD_HIGH_SLOT = 10077,/* req has bad highest_slot*/ + NFS4ERR_DEADSESSION = 10078,/*new req sent to dead sess*/ + NFS4ERR_ENCR_ALG_UNSUPP= 10079,/* encr alg. not supp. */ + NFS4ERR_PNFS_NO_LAYOUT = 10080,/* I/O without a layout */ + NFS4ERR_NOT_ONLY_OP = 10081,/* addl ops not allowed */ + NFS4ERR_WRONG_CRED = 10082,/* op done by wrong cred */ + NFS4ERR_WRONG_TYPE = 10083,/* op on wrong type object */ + NFS4ERR_DIRDELEG_UNAVAIL=10084,/* delegation not avail. */ + NFS4ERR_REJECT_DELEG = 10085,/* cb rejected delegation */ + NFS4ERR_RETURNCONFLICT = 10086,/* layout get before return*/ + NFS4ERR_DELEG_REVOKED = 10087, /* deleg./layout revoked */ + NFS4ERR_PARTNER_NOTSUPP = 10088, + NFS4ERR_PARTNER_NO_AUTH = 10089, + NFS4ERR_UNION_NOTSUPP = 10090, + NFS4ERR_OFFLOAD_DENIED = 10091, + NFS4ERR_WRONG_LFS = 10092, + NFS4ERR_BADLABEL = 10093, + NFS4ERR_OFFLOAD_NO_REQS = 10094, + NFS4ERR_NOXATTR = 10095, + NFS4ERR_XATTR2BIG = 10096 +}; /* * Basic data types */ +typedef opaque attrlist4<>; typedef uint32_t bitmap4<>; +typedef opaque verifier4[NFS4_VERIFIER_SIZE]; +typedef uint64_t nfs_cookie4; +typedef opaque nfs_fh4; typedef opaque utf8string<>; typedef utf8string utf8str_cis; typedef utf8string utf8str_cs; typedef utf8string utf8str_mixed; +typedef utf8str_cs component4; +typedef utf8str_cs linktext4; +typedef component4 pathname4<>; + /* * Timeval */ @@ -66,6 +209,21 @@ struct nfstime4 { uint32_t nseconds; }; +/* + * File attribute container + */ +struct fattr4 { + bitmap4 attrmask; + attrlist4 attr_vals; +}; + +/* + * Stateid + */ +struct stateid4 { + uint32_t seqid; + opaque other[12]; +}; /* * The following content was extracted from draft-ietf-nfsv4-delstid @@ -245,3 +403,82 @@ const FATTR4_ACL_TRUEFORM = 89; const FATTR4_ACL_TRUEFORM_SCOPE = 90; const FATTR4_POSIX_DEFAULT_ACL = 91; const FATTR4_POSIX_ACCESS_ACL = 92; + +/* + * Directory notification types. + */ +enum notify_type4 { + NOTIFY4_CHANGE_CHILD_ATTRS = 0, + NOTIFY4_CHANGE_DIR_ATTRS = 1, + NOTIFY4_REMOVE_ENTRY = 2, + NOTIFY4_ADD_ENTRY = 3, + NOTIFY4_RENAME_ENTRY = 4, + NOTIFY4_CHANGE_COOKIE_VERIFIER = 5 +}; + +/* Changed entry information. */ +struct notify_entry4 { + component4 ne_file; + fattr4 ne_attrs; +}; + +/* Previous entry information */ +struct prev_entry4 { + notify_entry4 pe_prev_entry; + /* what READDIR returned for this entry */ + nfs_cookie4 pe_prev_entry_cookie; +}; + +struct notify_remove4 { + notify_entry4 nrm_old_entry; + nfs_cookie4 nrm_old_entry_cookie; +}; + +struct notify_add4 { + /* + * Information on object + * possibly renamed over. + */ + notify_remove4 nad_old_entry<1>; + notify_entry4 nad_new_entry; + /* what READDIR would have returned for this entry */ + nfs_cookie4 nad_new_entry_cookie<1>; + prev_entry4 nad_prev_entry<1>; + bool nad_last_entry; +}; + +struct notify_attr4 { + notify_entry4 na_changed_entry; +}; + +struct notify_rename4 { + notify_remove4 nrn_old_entry; + notify_add4 nrn_new_entry; +}; + +struct notify_verifier4 { + verifier4 nv_old_cookieverf; + verifier4 nv_new_cookieverf; +}; + +/* + * Objects of type notify_<>4 and + * notify_device_<>4 are encoded in this. + */ +typedef opaque notifylist4<>; + +struct notify4 { + /* composed from notify_type4 or notify_deviceid_type4 */ + bitmap4 notify_mask; + notifylist4 notify_vals; +}; + +struct CB_NOTIFY4args { + stateid4 cna_stateid; + nfs_fh4 cna_fh; + notify4 cna_changes<>; +}; + +struct CB_NOTIFY4res { + nfsstat4 cnr_status; +}; diff --git a/Documentation/usb/gadget-testing.rst b/Documentation/usb/gadget-testing.rst index a6e8292f320a4b..4921e5307d493a 100644 --- a/Documentation/usb/gadget-testing.rst +++ b/Documentation/usb/gadget-testing.rst @@ -714,9 +714,6 @@ The uac1 function provides these attributes in its function directory: =============== ==================================== audio_buf_size audio buffer size - fn_cap capture pcm device file name - fn_cntl control device file name - fn_play playback pcm device file name req_buf_size ISO OUT endpoint request buffer size req_count ISO OUT endpoint request count =============== ==================================== diff --git a/Makefile b/Makefile index 408e8512b71345..73e7bac7527e69 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 7 PATCHLEVEL = 2 -SUBLEVEL = 5 +SUBLEVEL = 6 EXTRAVERSION = NAME = Baby Opossum Posse @@ -1361,7 +1361,7 @@ PHONY += vmlinux_o vmlinux_o: vmlinux.a $(KBUILD_VMLINUX_LIBS) $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.vmlinux_o -vmlinux.o modules.builtin.modinfo modules.builtin: vmlinux_o +vmlinux.o: vmlinux_o @: PHONY += vmlinux @@ -1605,10 +1605,10 @@ tools/%: FORCE PHONY += kselftest kselftest: headers - $(Q)$(MAKE) -C $(srctree)/tools/testing/selftests run_tests + $(Q)unset sub_make_done; $(MAKE) -C $(srctree)/tools/testing/selftests run_tests kselftest-%: headers FORCE - $(Q)$(MAKE) -C $(srctree)/tools/testing/selftests $* + $(Q)unset sub_make_done; $(MAKE) -C $(srctree)/tools/testing/selftests $* PHONY += kselftest-merge kselftest-merge: diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index b8aa3fcce1078e..bcb53bc9b931ae 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -133,6 +133,7 @@ config ARM select HAVE_PERF_EVENTS select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP + select HAVE_POSIX_CPU_TIMERS_TASK_WORK select MMU_GATHER_RCU_TABLE_FREE if SMP && ARM_LPAE select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RSEQ diff --git a/arch/arm/boot/dts/allwinner/sun4i-a10.dtsi b/arch/arm/boot/dts/allwinner/sun4i-a10.dtsi index 51a6464aab9a3b..cabf619c2e217f 100644 --- a/arch/arm/boot/dts/allwinner/sun4i-a10.dtsi +++ b/arch/arm/boot/dts/allwinner/sun4i-a10.dtsi @@ -185,7 +185,7 @@ pmu { compatible = "arm,cortex-a8-pmu"; - interrupts = <3>; + interrupts = <66>; }; reserved-memory { diff --git a/arch/arm/boot/dts/marvell/armada-388-helios4.dts b/arch/arm/boot/dts/marvell/armada-388-helios4.dts index ec134e22bae3e4..003b0708e96560 100644 --- a/arch/arm/boot/dts/marvell/armada-388-helios4.dts +++ b/arch/arm/boot/dts/marvell/armada-388-helios4.dts @@ -169,6 +169,7 @@ gpio-controller; #gpio-cells = <2>; reg = <0x20>; + vcc-supply = <®_3p3v>; pinctrl-names = "default"; pinctrl-0 = <&pca0_pins>; interrupt-parent = <&gpio0>; @@ -201,6 +202,10 @@ reg = <0x4c>; vcc-supply = <®_3p3v>; }; + + eeprom@53 { + vcc-supply = <®_3p3v>; + }; }; i2c@11100 { @@ -217,13 +222,17 @@ status = "okay"; #address-cells = <1>; #size-cells = <0>; + ahci-supply = <®_3p3v>; + phy-supply = <®_3p3v>; sata0: sata-port@0 { reg = <0>; + target-supply = <®_5p0v_hdd>; }; sata1: sata-port@1 { reg = <1>; + target-supply = <®_5p0v_hdd>; }; }; @@ -231,13 +240,17 @@ status = "okay"; #address-cells = <1>; #size-cells = <0>; + ahci-supply = <®_3p3v>; + phy-supply = <®_3p3v>; sata2: sata-port@0 { reg = <0>; + target-supply = <®_5p0v_hdd>; }; sata3: sata-port@1 { reg = <1>; + target-supply = <®_5p0v_hdd>; }; }; diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c index cd4b34c96e35e9..38feb30dfb5f86 100644 --- a/arch/arm/kernel/hw_breakpoint.c +++ b/arch/arm/kernel/hw_breakpoint.c @@ -929,10 +929,6 @@ static void hw_breakpoint_cfi_handler(struct pt_regs *regs) break; } } -#else -static void hw_breakpoint_cfi_handler(struct pt_regs *regs) -{ -} #endif /* @@ -964,9 +960,14 @@ static int hw_breakpoint_pending(unsigned long addr, unsigned int fsr, case ARM_ENTRY_SYNC_WATCHPOINT: watchpoint_handler(addr, fsr, regs); break; +#ifdef CONFIG_CFI case ARM_ENTRY_CFI_BREAKPOINT: - hw_breakpoint_cfi_handler(regs); + if (user_mode(regs)) + ret = 1; /* Don't handle userspace BKPT */ + else + hw_breakpoint_cfi_handler(regs); break; +#endif default: ret = 1; /* Unhandled fault. */ } diff --git a/arch/arm/mach-imx/avic.c b/arch/arm/mach-imx/avic.c index 3067c06b4b8ebd..6873a50bbe2c0a 100644 --- a/arch/arm/mach-imx/avic.c +++ b/arch/arm/mach-imx/avic.c @@ -173,6 +173,7 @@ static void __init mxc_init_irq(void __iomem *irqbase) np = of_find_compatible_node(NULL, NULL, "fsl,imx25-ccm"); mx25_ccm_base = of_iomap(np, 0); + of_node_put(np); if (mx25_ccm_base) { /* @@ -203,6 +204,7 @@ static void __init mxc_init_irq(void __iomem *irqbase) np = of_find_compatible_node(NULL, NULL, "fsl,avic"); domain = irq_domain_create_legacy(of_fwnode_handle(np), AVIC_NUM_IRQS, irq_base, 0, &irq_domain_simple_ops, NULL); + of_node_put(np); WARN_ON(!domain); for (i = 0; i < AVIC_NUM_IRQS / 32; i++, irq_base += 32) diff --git a/arch/arm/mach-imx/src.c b/arch/arm/mach-imx/src.c index 59a8e8cc44693b..c3c80b4c3d53bb 100644 --- a/arch/arm/mach-imx/src.c +++ b/arch/arm/mach-imx/src.c @@ -171,6 +171,7 @@ void __init imx_src_init(void) if (!np) return; src_base = of_iomap(np, 0); + of_node_put(np); WARN_ON(!src_base); /* @@ -195,6 +196,7 @@ void __init imx7_src_init(void) return; src_base = of_iomap(np, 0); + of_node_put(np); if (!src_base) return; @@ -203,6 +205,7 @@ void __init imx7_src_init(void) return; gpc_base = of_iomap(np, 0); + of_node_put(np); if (!gpc_base) return; } diff --git a/arch/arm/mach-lpc32xx/common.c b/arch/arm/mach-lpc32xx/common.c index 304ea61a07160b..35ed3569c5a350 100644 --- a/arch/arm/mach-lpc32xx/common.c +++ b/arch/arm/mach-lpc32xx/common.c @@ -106,7 +106,7 @@ void __init lpc32xx_map_io(void) iotable_init(lpc32xx_io_desc, ARRAY_SIZE(lpc32xx_io_desc)); } -static int __init lpc32xx_check_uid(void) +void __init lpc32xx_check_uid(void) { u32 uid[4]; @@ -119,7 +119,4 @@ static int __init lpc32xx_check_uid(void) system_serial_low = uid[0]; system_serial_high = uid[1]; } - - return 1; } -arch_initcall(lpc32xx_check_uid); diff --git a/arch/arm/mach-lpc32xx/common.h b/arch/arm/mach-lpc32xx/common.h index 32f0ad2178077a..06b20bea324e14 100644 --- a/arch/arm/mach-lpc32xx/common.h +++ b/arch/arm/mach-lpc32xx/common.h @@ -16,6 +16,8 @@ * Other arch specific structures and functions */ extern void __init lpc32xx_map_io(void); +extern void __init lpc32xx_check_uid(void); +extern void __init lpc32xx_pm_init(void); extern void __init lpc32xx_serial_init(void); /* diff --git a/arch/arm/mach-lpc32xx/phy3250.c b/arch/arm/mach-lpc32xx/phy3250.c index 66701bf432488b..ddc6333ca55da8 100644 --- a/arch/arm/mach-lpc32xx/phy3250.c +++ b/arch/arm/mach-lpc32xx/phy3250.c @@ -71,6 +71,8 @@ static const struct of_dev_auxdata lpc32xx_auxdata_lookup[] __initconst = { static void __init lpc3250_machine_init(void) { + lpc32xx_check_uid(); + lpc32xx_pm_init(); lpc32xx_serial_init(); of_platform_default_populate(NULL, lpc32xx_auxdata_lookup, NULL); diff --git a/arch/arm/mach-lpc32xx/pm.c b/arch/arm/mach-lpc32xx/pm.c index 2572bd89a5e8db..9b5c5e1462ed3b 100644 --- a/arch/arm/mach-lpc32xx/pm.c +++ b/arch/arm/mach-lpc32xx/pm.c @@ -120,7 +120,7 @@ static const struct platform_suspend_ops lpc32xx_pm_ops = { #define EMC_DYN_MEM_CTRL_OFS 0x20 #define EMC_SRMMC (1 << 3) #define EMC_CTRL_REG io_p2v(LPC32XX_EMC_BASE + EMC_DYN_MEM_CTRL_OFS) -static int __init lpc32xx_pm_init(void) +void __init lpc32xx_pm_init(void) { /* * Setup SDRAM self-refresh clock to automatically disable o @@ -129,7 +129,4 @@ static int __init lpc32xx_pm_init(void) __raw_writel(__raw_readl(EMC_CTRL_REG) | EMC_SRMMC, EMC_CTRL_REG); suspend_set_ops(&lpc32xx_pm_ops); - - return 0; } -arch_initcall(lpc32xx_pm_init); diff --git a/arch/arm/mach-omap2/omap_hwmod.c b/arch/arm/mach-omap2/omap_hwmod.c index 974107ff18b4ea..03cd523dff874b 100644 --- a/arch/arm/mach-omap2/omap_hwmod.c +++ b/arch/arm/mach-omap2/omap_hwmod.c @@ -2331,13 +2331,15 @@ static int __init _init(struct omap_hwmod *oh, void *data) if (r < 0) { WARN(1, "omap_hwmod: %s: doesn't have mpu register target base\n", oh->name); - return 0; + r = 0; + goto out_put_node; } r = _init_clocks(oh, np); if (r < 0) { WARN(1, "omap_hwmod: %s: couldn't init clocks\n", oh->name); - return -EINVAL; + r = -EINVAL; + goto out_put_node; } if (np) { @@ -2345,13 +2347,19 @@ static int __init _init(struct omap_hwmod *oh, void *data) parse_module_flags(oh, np); child = of_get_next_child(np, NULL); - if (child) + if (child) { parse_module_flags(oh, child); + of_node_put(child); + } } oh->_state = _HWMOD_STATE_INITIALIZED; - return 0; + r = 0; + +out_put_node: + of_node_put(bus); + return r; } /** @@ -3608,13 +3616,13 @@ int omap_hwmod_init_module(struct device *dev, #ifdef CONFIG_SERIAL_EARLYCON static void __init omap_hwmod_setup_earlycon_flags(void) { - struct device_node *np; + struct device_node *np, *chosen; struct omap_hwmod *oh; const char *uart; - np = of_find_node_by_path("/chosen"); - if (np) { - uart = of_get_property(np, "stdout-path", NULL); + chosen = of_find_node_by_path("/chosen"); + if (chosen) { + uart = of_get_property(chosen, "stdout-path", NULL); if (uart) { np = of_find_node_by_path(uart); if (np) { @@ -3629,8 +3637,10 @@ static void __init omap_hwmod_setup_earlycon_flags(void) if (oh) oh->flags |= DEBUG_OMAPUART_FLAGS; } + of_node_put(np); } } + of_node_put(chosen); } #endif diff --git a/arch/arm/mach-tegra/irq.c b/arch/arm/mach-tegra/irq.c index 4e1ee70b2a3f18..e5a611dce7e74b 100644 --- a/arch/arm/mach-tegra/irq.c +++ b/arch/arm/mach-tegra/irq.c @@ -66,9 +66,9 @@ static const struct of_device_id tegra114_dt_gic_match[] __initconst = { static void __init tegra114_gic_cpu_pm_registration(void) { - struct device_node *dn; + struct device_node *dn __free(device_node) = + of_find_matching_node(NULL, tegra114_dt_gic_match); - dn = of_find_matching_node(NULL, tegra114_dt_gic_match); if (!dn) return; @@ -88,7 +88,10 @@ static const struct of_device_id tegra_ictlr_match[] __initconst = { void __init tegra_init_irq(void) { - if (WARN_ON(!of_find_matching_node(NULL, tegra_ictlr_match))) + struct device_node *dn __free(device_node) = + of_find_matching_node(NULL, tegra_ictlr_match); + + if (WARN_ON(!dn)) pr_warn("Outdated DT detected, suspend/resume will NOT work\n"); tegra114_gic_cpu_pm_registration(); diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index e62cc4be5adf65..0a09d4ff771853 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -181,7 +181,11 @@ __do_user_fault(unsigned long addr, unsigned int fsr, unsigned int sig, pr_err("8<--- cut here ---\n"); pr_err("%s: unhandled page fault (%d) at 0x%08lx, code 0x%03x\n", tsk->comm, sig, addr, fsr); - show_pte(KERN_ERR, tsk->mm, addr); + if (likely(addr < TASK_SIZE)) { + mmap_write_lock(tsk->mm); + show_pte(KERN_ERR, tsk->mm, addr); + mmap_write_unlock(tsk->mm); + } show_regs(regs); } #endif @@ -633,10 +637,21 @@ do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs) if (!inf->fn(addr, fsr & ~FSR_LNX_PF, regs)) return; + if (likely(user_mode(regs))) + local_irq_enable(); + pr_alert("8<--- cut here ---\n"); pr_alert("Unhandled fault: %s (0x%03x) at 0x%08lx\n", inf->name, fsr, addr); - show_pte(KERN_ALERT, current->mm, addr); + if (likely(user_mode(regs))) { + if (addr < TASK_SIZE) { + mmap_write_lock(current->mm); + show_pte(KERN_ALERT, current->mm, addr); + mmap_write_unlock(current->mm); + } + } else { + show_pte(KERN_ALERT, current->mm, addr); + } arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr, fsr, 0); @@ -663,6 +678,9 @@ do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs) if (!inf->fn(addr, ifsr | FSR_LNX_PF, regs)) return; + if (likely(user_mode(regs))) + local_irq_enable(); + pr_alert("8<--- cut here ---\n"); pr_alert("Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n", inf->name, ifsr, addr); diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b3afe0688919bd..fc57d90d92c1d4 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2116,6 +2116,8 @@ config ARM64_BTI_KERNEL depends on !CC_IS_GCC || GCC_VERSION >= 100100 # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=106671 depends on !CC_IS_GCC + # https://github.com/llvm/llvm-project/issues/215547 + depends on !CC_IS_CLANG || CLANG_VERSION < 210000 depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_ARGS) help Build the kernel with Branch Target Identification annotations diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a64-pinephone.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-a64-pinephone.dtsi index 4bc6c1ef2cde43..f958bdbb0d3332 100644 --- a/arch/arm64/boot/dts/allwinner/sun50i-a64-pinephone.dtsi +++ b/arch/arm64/boot/dts/allwinner/sun50i-a64-pinephone.dtsi @@ -230,8 +230,8 @@ interrupts = <7 5 IRQ_TYPE_EDGE_RISING>; /* PH5 */ vdd-supply = <®_dldo1>; vddio-supply = <®_dldo1>; - mount-matrix = "0", "1", "0", - "-1", "0", "0", + mount-matrix = "0", "-1", "0", + "1", "0", "0", "0", "0", "1"; }; }; diff --git a/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts b/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts index 285c6ac1dd613c..2baf210a2a40e6 100644 --- a/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts +++ b/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts @@ -431,6 +431,10 @@ }; }; +&mipi_pcie_analog_dphy { + status = "okay"; +}; + &pdm { pinctrl-0 = <&pdm_dclk_a14_pins>, <&pdm_din0_pins>, <&pdm_din1_pins>, <&pdm_din2_pins>, <&pdm_din3_pins>; diff --git a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi index f1f53fd98ae257..b7a7f4fae7dc28 100644 --- a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi @@ -481,7 +481,8 @@ "nand_ale", "nand_cle", "nand_wen_clk", - "nand_ren_wr"; + "nand_ren_wr", + "nand_rb0"; function = "nand"; input-enable; bias-pull-up; diff --git a/arch/arm64/boot/dts/freescale/imx8-ss-audio.dtsi b/arch/arm64/boot/dts/freescale/imx8-ss-audio.dtsi index 5e4233ccfde469..f473d81f67ffaf 100644 --- a/arch/arm64/boot/dts/freescale/imx8-ss-audio.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8-ss-audio.dtsi @@ -124,10 +124,10 @@ audio_subsys: bus@59000000 { compatible = "fsl,imx8qm-asrc"; reg = <0x59000000 0x10000>; interrupts = ; - clocks = <&asrc0_lpcg IMX_LPCG_CLK_0>, - <&asrc0_lpcg IMX_LPCG_CLK_0>, - <&aud_pll_div0_lpcg IMX_LPCG_CLK_4>, - <&aud_pll_div1_lpcg IMX_LPCG_CLK_4>, + clocks = <&asrc0_lpcg IMX_LPCG_CLK_4>, + <&asrc0_lpcg IMX_LPCG_CLK_4>, + <&aud_pll_div0_lpcg IMX_LPCG_CLK_0>, + <&aud_pll_div1_lpcg IMX_LPCG_CLK_0>, <&acm IMX_ADMA_ACM_AUD_CLK0_SEL>, <&acm IMX_ADMA_ACM_AUD_CLK1_SEL>, <&clk_dummy>, diff --git a/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts b/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts index 443e4fd5b9bfc1..285bf79864ebdc 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts @@ -775,6 +775,10 @@ status = "okay"; }; +&mu2 { + status = "okay"; +}; + &pwm1 { pinctrl-0 = <&pinctrl_pwm1>; pinctrl-names = "default"; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-frdm.dts b/arch/arm64/boot/dts/freescale/imx8mp-frdm.dts index 5fb9714215bfe6..f43330d1ff8b63 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-frdm.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-frdm.dts @@ -562,6 +562,8 @@ pinctrl_hdmi: hdmigrp { fsl,pins = < + MX8MP_IOMUXC_HDMI_DDC_SCL__HDMIMIX_HDMI_SCL 0x1c2 + MX8MP_IOMUXC_HDMI_DDC_SDA__HDMIMIX_HDMI_SDA 0x1c2 MX8MP_IOMUXC_HDMI_CEC__HDMIMIX_HDMI_CEC 0x10 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi b/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi index c79b1df339db1e..f881912cde4605 100644 --- a/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi +++ b/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi @@ -15,9 +15,9 @@ rtc1 = &bbnsm_rtc; }; - memory@40000000 { + memory@80000000 { device_type = "memory"; - reg = <0x0 0x40000000 0 0x80000000>; + reg = <0x0 0x80000000 0 0x40000000>; }; chosen { diff --git a/arch/arm64/boot/dts/freescale/imx94.dtsi b/arch/arm64/boot/dts/freescale/imx94.dtsi index a6cb5a6e848b3e..1f9035e6cf159a 100644 --- a/arch/arm64/boot/dts/freescale/imx94.dtsi +++ b/arch/arm64/boot/dts/freescale/imx94.dtsi @@ -1374,8 +1374,9 @@ <0 0x4c360000 0 0x10000>, <0 0x4c340000 0 0x4000>; reg-names = "dbi", "config", "atu", "app"; - ranges = <0x81000000 0x0 0x00000000 0x0 0x6ff00000 0 0x00100000>, - <0x82000000 0x0 0x10000000 0x9 0x10000000 0 0x80000000>; + ranges = <0x43000000 0x9 0x00000000 0x9 0x00000000 0x0 0xe0000000>, + <0x82000000 0x0 0xe0000000 0x9 0xe0000000 0x0 0x10000000>, + <0x81000000 0x0 0x00000000 0x9 0xf0000000 0x0 0x00100000>; #address-cells = <3>; #size-cells = <2>; device_type = "pci"; diff --git a/arch/arm64/boot/dts/freescale/imx943.dtsi b/arch/arm64/boot/dts/freescale/imx943.dtsi index ed030d4bc7bd97..cf5b3dbb47ff75 100644 --- a/arch/arm64/boot/dts/freescale/imx943.dtsi +++ b/arch/arm64/boot/dts/freescale/imx943.dtsi @@ -218,8 +218,9 @@ <0 0x4c3e0000 0 0x10000>, <0 0x4c3c0000 0 0x4000>; reg-names = "dbi", "config", "atu", "app"; - ranges = <0x81000000 0 0x00000000 0x8 0x8ff00000 0 0x00100000>, - <0x82000000 0 0x10000000 0xa 0x10000000 0 0x80000000>; + ranges = <0x43000000 0xa 0x00000000 0xa 0x00000000 0x0 0xe0000000>, + <0x82000000 0x0 0xe0000000 0xa 0xe0000000 0x0 0x10000000>, + <0x81000000 0x0 0x00000000 0xa 0xf0000000 0x0 0x00100000>; #address-cells = <3>; #size-cells = <2>; device_type = "pci"; diff --git a/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi b/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi index 7d760470201fa2..a6c5398a81e3c7 100644 --- a/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi +++ b/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi @@ -24,6 +24,7 @@ serial1 = &lpuart1; serial2 = &lpuart6; serial3 = &lpuart3; + serial4 = &lpuart5; }; chosen { diff --git a/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts b/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts index f4d73c8b1a6d31..bd00e7df87c67c 100644 --- a/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts +++ b/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts @@ -290,7 +290,8 @@ &usb3 { status = "okay"; - phys = <&comphy2 0>; + phys = <&usb2_utmi_otg_phy>, <&comphy2 0>; + phy-names = "usb2-phy", "usb3-phy"; }; &mdio { diff --git a/arch/arm64/boot/dts/mediatek/mt8390-tungsten-smarc.dtsi b/arch/arm64/boot/dts/mediatek/mt8390-tungsten-smarc.dtsi index 9f5a0ec563e8c9..8256279c562994 100644 --- a/arch/arm64/boot/dts/mediatek/mt8390-tungsten-smarc.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8390-tungsten-smarc.dtsi @@ -265,8 +265,6 @@ }; &disp_dsi0 { - #address-cells = <1>; - #size-cells = <0>; status = "okay"; ports { @@ -1049,8 +1047,6 @@ ethernet@1 { compatible = "usb424,7850"; reg = <1>; - #address-cells = <1>; - #size-cells = <0>; mdio { #address-cells = <1>; diff --git a/arch/arm64/boot/dts/nvidia/tegra264.dtsi b/arch/arm64/boot/dts/nvidia/tegra264.dtsi index 2d2cb1a3d95cb2..bcbf0a558d8db5 100644 --- a/arch/arm64/boot/dts/nvidia/tegra264.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra264.dtsi @@ -3393,7 +3393,7 @@ cmdqv1: cmdqv@5200000 { compatible = "nvidia,tegra264-cmdqv"; reg = <0x00 0x5200000 0x0 0x830000>; - interrupts = ; + interrupts = ; status = "disabled"; }; @@ -3413,7 +3413,7 @@ cmdqv2: cmdqv@6200000 { compatible = "nvidia,tegra264-cmdqv"; reg = <0x00 0x6200000 0x0 0x830000>; - interrupts = ; + interrupts = ; status = "disabled"; }; @@ -3486,7 +3486,7 @@ cmdqv0: cmdqv@a200000 { compatible = "nvidia,tegra264-cmdqv"; reg = <0x00 0xa200000 0x0 0x830000>; - interrupts = ; + interrupts = ; status = "disabled"; }; @@ -3506,7 +3506,7 @@ cmdqv4: cmdqv@b200000 { compatible = "nvidia,tegra264-cmdqv"; reg = <0x00 0xb200000 0x0 0x830000>; - interrupts = ; + interrupts = ; status = "disabled"; }; @@ -3831,7 +3831,7 @@ cmdqv3: cmdqv@6200000 { compatible = "nvidia,tegra264-cmdqv"; reg = <0x00 0x6200000 0x0 0x830000>; - interrupts = ; + interrupts = ; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/qcom/agatti.dtsi b/arch/arm64/boot/dts/qcom/agatti.dtsi index f0b6ae9b81528a..590bd2432d85ba 100644 --- a/arch/arm64/boot/dts/qcom/agatti.dtsi +++ b/arch/arm64/boot/dts/qcom/agatti.dtsi @@ -2197,6 +2197,7 @@ "gcc_disp_gpll0_div_clk_src", "dsi0_phy_pll_out_byteclk", "dsi0_phy_pll_out_dsiclk"; + power-domains = <&rpmpd QCM2290_VDDCX>; #power-domain-cells = <1>; #clock-cells = <1>; #reset-cells = <1>; diff --git a/arch/arm64/boot/dts/qcom/eliza-mtp.dts b/arch/arm64/boot/dts/qcom/eliza-mtp.dts index 1374afd9d14e36..9d58ef20213a05 100644 --- a/arch/arm64/boot/dts/qcom/eliza-mtp.dts +++ b/arch/arm64/boot/dts/qcom/eliza-mtp.dts @@ -535,8 +535,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3g>; - vdda-pll-supply = <&vreg_l7k>; + vdda-phy-supply = <&vreg_l7k>; + vdda-pll-supply = <&vreg_l3g>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/eliza.dtsi b/arch/arm64/boot/dts/qcom/eliza.dtsi index 72b786fec195f2..8fa69be1505087 100644 --- a/arch/arm64/boot/dts/qcom/eliza.dtsi +++ b/arch/arm64/boot/dts/qcom/eliza.dtsi @@ -1232,8 +1232,6 @@ #size-cells = <2>; ranges; - status = "disabled"; - i2c0: i2c@a80000 { compatible = "qcom,geni-i2c"; reg = <0x0 0x00a80000 0x0 0x4000>; @@ -1969,7 +1967,7 @@ }; remoteproc_adsp: remoteproc@3000000 { - compatible = "qcom,eliza-adsp-pas"; + compatible = "qcom,eliza-adsp-pas", "qcom,sm8550-adsp-pas"; reg = <0x0 0x03000000 0x0 0x10000>; interrupts-extended = <&pdc 6 IRQ_TYPE_EDGE_RISING>, diff --git a/arch/arm64/boot/dts/qcom/glymur-crd.dts b/arch/arm64/boot/dts/qcom/glymur-crd.dts index c98dfb3941fa36..6125617de82a66 100644 --- a/arch/arm64/boot/dts/qcom/glymur-crd.dts +++ b/arch/arm64/boot/dts/qcom/glymur-crd.dts @@ -8,198 +8,9 @@ #include "glymur.dtsi" #include "glymur-crd.dtsi" -#include - / { model = "Qualcomm Technologies, Inc. Glymur CRD"; compatible = "qcom,glymur-crd", "qcom,glymur"; - - pmic-glink { - compatible = "qcom,glymur-pmic-glink", - "qcom,pmic-glink"; - #address-cells = <1>; - #size-cells = <0>; - - connector@0 { - compatible = "usb-c-connector"; - reg = <0>; - power-role = "dual"; - data-role = "dual"; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - - pmic_glink_hs_in: endpoint { - remote-endpoint = <&usb_0_dwc3_hs>; - }; - }; - - port@1 { - reg = <1>; - - pmic_glink_ss_in: endpoint { - remote-endpoint = <&usb_0_qmpphy_out>; - }; - }; - }; - }; - - connector@1 { - compatible = "usb-c-connector"; - reg = <1>; - power-role = "dual"; - data-role = "dual"; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - - pmic_glink_hs_in1: endpoint { - remote-endpoint = <&usb_1_dwc3_hs>; - }; - }; - - port@1 { - reg = <1>; - - pmic_glink_ss_in1: endpoint { - remote-endpoint = <&usb_1_qmpphy_out>; - }; - }; - }; - }; - }; - - vreg_edp_3p3: regulator-edp-3p3 { - compatible = "regulator-fixed"; - - regulator-name = "VREG_EDP_3P3"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - - gpio = <&tlmm 70 GPIO_ACTIVE_HIGH>; - enable-active-high; - - pinctrl-0 = <&edp_reg_en>; - pinctrl-names = "default"; - - regulator-boot-on; - }; - - vreg_misc_3p3: regulator-misc-3p3 { - compatible = "regulator-fixed"; - - regulator-name = "VREG_MISC_3P3"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - - gpio = <&pmh0110_f_e0_gpios 6 GPIO_ACTIVE_HIGH>; - enable-active-high; - - pinctrl-0 = <&misc_3p3_reg_en>; - pinctrl-names = "default"; - - regulator-boot-on; - }; -}; - -&i2c0 { - clock-frequency = <400000>; - - status = "okay"; - - touchpad@2c { - compatible = "hid-over-i2c"; - reg = <0x2c>; - - hid-descr-addr = <0x20>; - interrupts-extended = <&tlmm 3 IRQ_TYPE_LEVEL_LOW>; - - vdd-supply = <&vreg_misc_3p3>; - vddl-supply = <&vreg_l15b_e0_1p8>; - - pinctrl-0 = <&tpad_default>; - pinctrl-names = "default"; - - wakeup-source; - }; - - keyboard@3a { - compatible = "hid-over-i2c"; - reg = <0x3a>; - - hid-descr-addr = <0x1>; - interrupts-extended = <&tlmm 67 IRQ_TYPE_LEVEL_LOW>; - - vdd-supply = <&vreg_misc_3p3>; - vddl-supply = <&vreg_l15b_e0_1p8>; - - pinctrl-0 = <&kybd_default>; - pinctrl-names = "default"; - - wakeup-source; - }; -}; - -&i2c8 { - clock-frequency = <400000>; - - status = "okay"; - - touchscreen@38 { - compatible = "hid-over-i2c"; - reg = <0x38>; - - hid-descr-addr = <0x1>; - interrupts-extended = <&tlmm 51 IRQ_TYPE_LEVEL_LOW>; - - vdd-supply = <&vreg_misc_3p3>; - vddl-supply = <&vreg_l15b_e0_1p8>; - - pinctrl-0 = <&ts0_default>; - pinctrl-names = "default"; - }; -}; - -&i2c5 { - clock-frequency = <400000>; - - status = "okay"; - - ptn3222_0: redriver@43 { - compatible = "nxp,ptn3222"; - reg = <0x43>; - - reset-gpios = <&tlmm 8 GPIO_ACTIVE_LOW>; - - vdd3v3-supply = <&vreg_l8b_e0_1p50>; - vdd1v8-supply = <&vreg_l15b_e0_1p8>; - - #phy-cells = <0>; - }; - - ptn3222_1: redriver@47 { - compatible = "nxp,ptn3222"; - reg = <0x47>; - - reset-gpios = <&tlmm 9 GPIO_ACTIVE_LOW>; - - vdd3v3-supply = <&vreg_l8b_e0_1p50>; - vdd1v8-supply = <&vreg_l15b_e0_1p8>; - - #phy-cells = <0>; - }; -}; - -&mdss { - status = "okay"; }; &mdss_dp0 { @@ -217,213 +28,3 @@ &mdss_dp1_out { link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>; }; - -&mdss_dp3 { - /delete-property/ #sound-dai-cells; - - status = "okay"; - - aux-bus { - panel { - compatible = "samsung,atna60cl08", "samsung,atna33xc20"; - enable-gpios = <&tlmm 18 GPIO_ACTIVE_HIGH>; - power-supply = <&vreg_edp_3p3>; - - pinctrl-0 = <&edp_bl_en>; - pinctrl-names = "default"; - - port { - edp_panel_in: endpoint { - remote-endpoint = <&mdss_dp3_out>; - }; - }; - }; - }; -}; - -&mdss_dp3_out { - data-lanes = <0 1 2 3>; - link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>; - - remote-endpoint = <&edp_panel_in>; -}; - -&mdss_dp3_phy { - vdda-phy-supply = <&vreg_l2f_e1_0p83>; - vdda-pll-supply = <&vreg_l4f_e1_1p08>; - - status = "okay"; -}; - -&pmh0110_f_e0_gpios { - misc_3p3_reg_en: misc-3p3-reg-en-state { - pins = "gpio6"; - function = "normal"; - bias-disable; - input-disable; - output-enable; - drive-push-pull; - power-source = <1>; /* 1.8 V */ - qcom,drive-strength = ; - }; -}; - -&smb2370_j_e2_eusb2_repeater { - vdd18-supply = <&vreg_l15b_e0_1p8>; - vdd3-supply = <&vreg_l7b_e0_2p79>; -}; - -&smb2370_k_e2_eusb2_repeater { - vdd18-supply = <&vreg_l15b_e0_1p8>; - vdd3-supply = <&vreg_l7b_e0_2p79>; -}; - -&tlmm { - edp_bl_en: edp-bl-en-state { - pins = "gpio18"; - function = "gpio"; - drive-strength = <16>; - bias-disable; - }; - - edp_reg_en: edp-reg-en-state { - pins = "gpio70"; - function = "gpio"; - drive-strength = <16>; - bias-disable; - }; - - kybd_default: kybd-default-state { - pins = "gpio67"; - function = "gpio"; - bias-disable; - }; - - tpad_default: tpad-default-state { - pins = "gpio3"; - function = "gpio"; - bias-disable; - }; - - ts0_default: ts0-default-state { - int-n-pins { - pins = "gpio51"; - function = "gpio"; - bias-disable; - }; - - reset-n-pins { - pins = "gpio48"; - function = "gpio"; - drive-strength = <16>; - bias-disable; - }; - }; -}; - -&usb_0 { - status = "okay"; -}; - -&usb_0_dwc3_hs { - remote-endpoint = <&pmic_glink_hs_in>; -}; - -&usb_0_hsphy { - vdd-supply = <&vreg_l3f_e0_0p72>; - vdda12-supply = <&vreg_l4h_e0_1p2>; - - phys = <&smb2370_j_e2_eusb2_repeater>; - - status = "okay"; -}; - -&usb_0_qmpphy { - vdda-phy-supply = <&vreg_l4h_e0_1p2>; - vdda-pll-supply = <&vreg_l3f_e0_0p72>; - refgen-supply = <&vreg_l2f_e0_0p82>; - - status = "okay"; -}; - -&usb_0_qmpphy_out { - remote-endpoint = <&pmic_glink_ss_in>; -}; - -&usb_1 { - status = "okay"; -}; - -&usb_1_dwc3_hs { - remote-endpoint = <&pmic_glink_hs_in1>; -}; - -&usb_1_hsphy { - vdd-supply = <&vreg_l3f_e0_0p72>; - vdda12-supply = <&vreg_l4h_e0_1p2>; - - phys = <&smb2370_k_e2_eusb2_repeater>; - - status = "okay"; -}; - -&usb_1_qmpphy { - vdda-phy-supply = <&vreg_l4h_e0_1p2>; - vdda-pll-supply = <&vreg_l1h_e0_0p89>; - refgen-supply = <&vreg_l2f_e0_0p82>; - - status = "okay"; -}; - -&usb_1_qmpphy_out { - remote-endpoint = <&pmic_glink_ss_in1>; -}; - -&usb_hs { - status = "okay"; -}; - -&usb_hs_phy { - vdd-supply = <&vreg_l2h_e0_0p72>; - vdda12-supply = <&vreg_l4h_e0_1p2>; - - phys = <&ptn3222_1>; - - status = "okay"; -}; - -&usb_mp { - status = "okay"; -}; - -&usb_mp_hsphy0 { - vdd-supply = <&vreg_l2h_e0_0p72>; - vdda12-supply = <&vreg_l4h_e0_1p2>; - - phys = <&ptn3222_0>; - - status = "okay"; -}; - -&usb_mp_hsphy1 { - vdd-supply = <&vreg_l2h_e0_0p72>; - vdda12-supply = <&vreg_l4h_e0_1p2>; - - status = "okay"; -}; - -&usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l4h_e0_1p2>; - vdda-pll-supply = <&vreg_l2h_e0_0p72>; - refgen-supply = <&vreg_l4f_e1_1p08>; - - status = "okay"; -}; - -&usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l4h_e0_1p2>; - vdda-pll-supply = <&vreg_l2h_e0_0p72>; - refgen-supply = <&vreg_l4f_e1_1p08>; - - status = "okay"; -}; diff --git a/arch/arm64/boot/dts/qcom/glymur-crd.dtsi b/arch/arm64/boot/dts/qcom/glymur-crd.dtsi index e784b538f42e13..79548f6f03f3f2 100644 --- a/arch/arm64/boot/dts/qcom/glymur-crd.dtsi +++ b/arch/arm64/boot/dts/qcom/glymur-crd.dtsi @@ -11,6 +11,7 @@ #include "smb2370.dtsi" /* SPMI2: SID-9/10/11 */ #include +#include / { model = "Qualcomm Technologies, Inc. Glymur CRD"; @@ -22,6 +23,7 @@ i2c0 = &i2c0; i2c1 = &i2c4; i2c2 = &i2c5; + i2c3 = &i2c9; spi0 = &spi18; }; @@ -68,6 +70,101 @@ }; }; + pmic-glink { + compatible = "qcom,glymur-pmic-glink", + "qcom,pmic-glink"; + #address-cells = <1>; + #size-cells = <0>; + + connector@0 { + compatible = "usb-c-connector"; + reg = <0>; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + pmic_glink_hs_in: endpoint { + remote-endpoint = <&usb_0_dwc3_hs>; + }; + }; + + port@1 { + reg = <1>; + + pmic_glink_ss_in: endpoint { + remote-endpoint = <&usb_0_qmpphy_out>; + }; + }; + }; + }; + + connector@1 { + compatible = "usb-c-connector"; + reg = <1>; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + pmic_glink_hs_in1: endpoint { + remote-endpoint = <&usb_1_dwc3_hs>; + }; + }; + + port@1 { + reg = <1>; + + pmic_glink_ss_in1: endpoint { + remote-endpoint = <&usb_1_qmpphy_out>; + }; + }; + }; + }; + }; + + vreg_edp_3p3: regulator-edp-3p3 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_EDP_3P3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&tlmm 70 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&edp_reg_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_misc_3p3: regulator-misc-3p3 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_MISC_3P3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&pmh0110_f_e0_gpios 6 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&misc_3p3_reg_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + vreg_nvme: regulator-nvme { compatible = "regulator-fixed"; @@ -444,6 +541,152 @@ }; }; +&i2c9 { + clock-frequency = <400000>; + + status = "okay"; + + embedded-controller@76 { + compatible = "qcom,glymur-crd-ec", "qcom,hamoa-crd-ec"; + reg = <0x76>; + + interrupts-extended = <&tlmm 66 IRQ_TYPE_EDGE_FALLING>; + + pinctrl-0 = <&ec_int_n_default>; + pinctrl-names = "default"; + }; +}; + +&i2c0 { + clock-frequency = <400000>; + + status = "okay"; + + touchpad@2c { + compatible = "hid-over-i2c"; + reg = <0x2c>; + + hid-descr-addr = <0x20>; + interrupts-extended = <&tlmm 3 IRQ_TYPE_LEVEL_LOW>; + + vdd-supply = <&vreg_misc_3p3>; + vddl-supply = <&vreg_l15b_e0_1p8>; + + pinctrl-0 = <&tpad_default>; + pinctrl-names = "default"; + + wakeup-source; + }; + + keyboard@3a { + compatible = "hid-over-i2c"; + reg = <0x3a>; + + hid-descr-addr = <0x1>; + interrupts-extended = <&tlmm 67 IRQ_TYPE_LEVEL_LOW>; + + vdd-supply = <&vreg_misc_3p3>; + vddl-supply = <&vreg_l15b_e0_1p8>; + + pinctrl-0 = <&kybd_default>; + pinctrl-names = "default"; + + wakeup-source; + }; +}; + +&i2c5 { + clock-frequency = <400000>; + + status = "okay"; + + ptn3222_0: redriver@43 { + compatible = "nxp,ptn3222"; + reg = <0x43>; + + reset-gpios = <&tlmm 8 GPIO_ACTIVE_LOW>; + + vdd3v3-supply = <&vreg_l8b_e0_1p50>; + vdd1v8-supply = <&vreg_l15b_e0_1p8>; + + #phy-cells = <0>; + }; + + ptn3222_1: redriver@47 { + compatible = "nxp,ptn3222"; + reg = <0x47>; + + reset-gpios = <&tlmm 9 GPIO_ACTIVE_LOW>; + + vdd3v3-supply = <&vreg_l8b_e0_1p50>; + vdd1v8-supply = <&vreg_l15b_e0_1p8>; + + #phy-cells = <0>; + }; +}; + +&i2c8 { + clock-frequency = <400000>; + + status = "okay"; + + touchscreen@38 { + compatible = "focaltech,ft3d81", "focaltech,ft8112"; + reg = <0x38>; + + interrupts-extended = <&tlmm 51 IRQ_TYPE_LEVEL_LOW>; + + vcc33-supply = <&vreg_misc_3p3>; + vccio-supply = <&vreg_l15b_e0_1p8>; + + reset-gpios = <&tlmm 48 GPIO_ACTIVE_LOW>; + + pinctrl-0 = <&ts0_default>; + pinctrl-names = "default"; + }; +}; + +&mdss { + status = "okay"; +}; + +&mdss_dp3 { + /delete-property/ #sound-dai-cells; + + status = "okay"; + + aux-bus { + panel { + compatible = "samsung,atna60cl08", "samsung,atna33xc20"; + enable-gpios = <&tlmm 18 GPIO_ACTIVE_HIGH>; + power-supply = <&vreg_edp_3p3>; + + pinctrl-0 = <&edp_bl_en>; + pinctrl-names = "default"; + + port { + edp_panel_in: endpoint { + remote-endpoint = <&mdss_dp3_out>; + }; + }; + }; + }; +}; + +&mdss_dp3_out { + data-lanes = <0 1 2 3>; + link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>; + + remote-endpoint = <&edp_panel_in>; +}; + +&mdss_dp3_phy { + vdda-phy-supply = <&vreg_l2f_e1_0p83>; + vdda-pll-supply = <&vreg_l4f_e1_1p08>; + + status = "okay"; +}; + &pcie3b { vddpe-3v3-supply = <&vreg_nvmesec>; @@ -537,14 +780,6 @@ wake-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>; }; -&pmh0101_gpios { - nvme_reg_en: nvme-reg-en-state { - pins = "gpio14"; - function = "normal"; - bias-disable; - }; -}; - &pmh0110_f_e1_gpios { nvme_sec_reg_en: nvme-reg-en-state { pins = "gpio14"; @@ -560,6 +795,25 @@ output-disable; bias-pull-up; }; + + nvme_reg_en: nvme-reg-en-state { + pins = "gpio14"; + function = "normal"; + bias-disable; + }; +}; + +&pmh0110_f_e0_gpios { + misc_3p3_reg_en: misc-3p3-reg-en-state { + pins = "gpio6"; + function = "normal"; + bias-disable; + input-disable; + output-enable; + drive-push-pull; + power-source = <1>; /* 1.8 V */ + qcom,drive-strength = ; + }; }; &pmk8850_rtc { @@ -585,17 +839,53 @@ status = "okay"; }; +&smb2370_j_e2_eusb2_repeater { + vdd18-supply = <&vreg_l15b_e0_1p8>; + vdd3-supply = <&vreg_l7b_e0_2p79>; +}; + +&smb2370_k_e2_eusb2_repeater { + vdd18-supply = <&vreg_l15b_e0_1p8>; + vdd3-supply = <&vreg_l7b_e0_2p79>; +}; + &tlmm { gpio-reserved-ranges = <4 4>, /* EC TZ Secure I3C */ <10 2>, /* OOB UART */ <44 4>; /* Security SPI (TPM) */ + edp_bl_en: edp-bl-en-state { + pins = "gpio18"; + function = "gpio"; + drive-strength = <16>; + bias-disable; + }; + + edp_reg_en: edp-reg-en-state { + pins = "gpio70"; + function = "gpio"; + drive-strength = <16>; + bias-disable; + }; + hall_int_n_default: hall-int-n-state { pins = "gpio92"; function = "gpio"; bias-disable; }; + ec_int_n_default: ec-int-n-state { + pins = "gpio66"; + function = "gpio"; + bias-disable; + }; + + kybd_default: kybd-default-state { + pins = "gpio67"; + function = "gpio"; + bias-disable; + }; + pcie4_default: pcie4-default-state { clkreq-n-pins { pins = "gpio147"; @@ -688,6 +978,27 @@ }; }; + tpad_default: tpad-default-state { + pins = "gpio3"; + function = "gpio"; + bias-disable; + }; + + ts0_default: ts0-default-state { + int-n-pins { + pins = "gpio51"; + function = "gpio"; + bias-disable; + }; + + reset-n-pins { + pins = "gpio48"; + function = "gpio"; + drive-strength = <16>; + bias-disable; + }; + }; + wcn_wlan_bt_en: wcn-wlan-bt-en-state { pins = "gpio116", "gpio117"; function = "gpio"; @@ -726,3 +1037,110 @@ vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>; }; }; + +&usb_0 { + status = "okay"; +}; + +&usb_0_dwc3_hs { + remote-endpoint = <&pmic_glink_hs_in>; +}; + +&usb_0_hsphy { + vdd-supply = <&vreg_l3f_e0_0p72>; + vdda12-supply = <&vreg_l4h_e0_1p2>; + + phys = <&smb2370_j_e2_eusb2_repeater>; + + status = "okay"; +}; + +&usb_0_qmpphy { + vdda-phy-supply = <&vreg_l3f_e0_0p72>; + vdda-pll-supply = <&vreg_l4h_e0_1p2>; + refgen-supply = <&vreg_l2f_e0_0p82>; + + status = "okay"; +}; + +&usb_0_qmpphy_out { + remote-endpoint = <&pmic_glink_ss_in>; +}; + +&usb_1 { + status = "okay"; +}; + +&usb_1_dwc3_hs { + remote-endpoint = <&pmic_glink_hs_in1>; +}; + +&usb_1_hsphy { + vdd-supply = <&vreg_l3f_e0_0p72>; + vdda12-supply = <&vreg_l4h_e0_1p2>; + + phys = <&smb2370_k_e2_eusb2_repeater>; + + status = "okay"; +}; + +&usb_1_qmpphy { + vdda-phy-supply = <&vreg_l1h_e0_0p89>; + vdda-pll-supply = <&vreg_l4h_e0_1p2>; + refgen-supply = <&vreg_l2f_e0_0p82>; + + status = "okay"; +}; + +&usb_1_qmpphy_out { + remote-endpoint = <&pmic_glink_ss_in1>; +}; + +&usb_hs { + status = "okay"; +}; + +&usb_hs_phy { + vdd-supply = <&vreg_l2h_e0_0p72>; + vdda12-supply = <&vreg_l4h_e0_1p2>; + + phys = <&ptn3222_1>; + + status = "okay"; +}; + +&usb_mp { + status = "okay"; +}; + +&usb_mp_hsphy0 { + vdd-supply = <&vreg_l2h_e0_0p72>; + vdda12-supply = <&vreg_l4h_e0_1p2>; + + phys = <&ptn3222_0>; + + status = "okay"; +}; + +&usb_mp_hsphy1 { + vdd-supply = <&vreg_l2h_e0_0p72>; + vdda12-supply = <&vreg_l4h_e0_1p2>; + + status = "okay"; +}; + +&usb_mp_qmpphy0 { + vdda-phy-supply = <&vreg_l2h_e0_0p72>; + vdda-pll-supply = <&vreg_l4h_e0_1p2>; + refgen-supply = <&vreg_l4f_e1_1p08>; + + status = "okay"; +}; + +&usb_mp_qmpphy1 { + vdda-phy-supply = <&vreg_l2h_e0_0p72>; + vdda-pll-supply = <&vreg_l4h_e0_1p2>; + refgen-supply = <&vreg_l4f_e1_1p08>; + + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/qcom/glymur.dtsi b/arch/arm64/boot/dts/qcom/glymur.dtsi index 129be417ac4245..16f51854748110 100644 --- a/arch/arm64/boot/dts/qcom/glymur.dtsi +++ b/arch/arm64/boot/dts/qcom/glymur.dtsi @@ -602,13 +602,13 @@ no-map; }; - soccpdtb_mem: soccpdtb@892e0000 { - reg = <0x0 0x892e0000 0x0 0x20000>; + soccp_mem: soccp@88e00000 { + reg = <0x0 0x88e00000 0x0 0x400000>; no-map; }; - soccp_mem: soccp@89300000 { - reg = <0x0 0x89300000 0x0 0x400000>; + soccpdtb_mem: soccpdtb@89200000 { + reg = <0x0 0x89200000 0x0 0x20000>; no-map; }; @@ -786,8 +786,8 @@ <&usb_0_qmpphy QMP_USB43DP_USB3_PIPE_CLK>, <&usb_1_qmpphy QMP_USB43DP_USB3_PIPE_CLK>, <&usb_2_qmpphy QMP_USB43DP_USB3_PIPE_CLK>, - <&usb_mp_qmpphy0 QMP_USB43DP_USB3_PIPE_CLK>, - <&usb_mp_qmpphy1 QMP_USB43DP_USB3_PIPE_CLK>, + <&usb_mp_qmpphy0>, /* USB3 UNI PHY pipe 0 */ + <&usb_mp_qmpphy1>, /* USB3 UNI PHY pipe 1 */ <0>, /* USB4 PHY 0 pcie pipe */ <0>, /* USB4 PHY 0 Max pipe */ <0>, /* USB4 PHY 1 pcie pipe */ @@ -797,6 +797,7 @@ #clock-cells = <1>; #reset-cells = <1>; #power-domain-cells = <1>; + power-domains = <&rpmhpd RPMHPD_CX>; }; gpi_dma2: dma-controller@800000 { @@ -4835,9 +4836,13 @@ pdc: interrupt-controller@b220000 { compatible = "qcom,glymur-pdc", "qcom,pdc"; reg = <0x0 0x0b220000 0x0 0x10000>; - qcom,pdc-ranges = <0 745 51>, - <51 527 47>, - <98 609 32>, + qcom,pdc-ranges = <0 745 38>, + <40 785 11>, + <51 527 4>, + <57 533 10>, + <70 546 4>, + <75 551 18>, + <108 619 22>, <130 717 12>, <142 251 5>, <147 796 16>, @@ -5029,7 +5034,7 @@ #size-cells = <0>; }; - spmi_bus2: spmi@c48000 { + spmi_bus2: spmi@c448000 { reg = <0x0 0x0c448000 0x0 0x4000>, <0x0 0x0c8e0000 0x0 0x10000>, <0x0 0x0c44c000 0x0 0x8000>; diff --git a/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi b/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi index 9c5e77df005476..bc70fe5db96f5b 100644 --- a/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi +++ b/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi @@ -590,8 +590,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -611,8 +611,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -632,8 +632,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -670,15 +670,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/hamoa-lenovo-ideacentre-mini-01q8x10.dts b/arch/arm64/boot/dts/qcom/hamoa-lenovo-ideacentre-mini-01q8x10.dts index bfb7cea56df964..28482500838913 100644 --- a/arch/arm64/boot/dts/qcom/hamoa-lenovo-ideacentre-mini-01q8x10.dts +++ b/arch/arm64/boot/dts/qcom/hamoa-lenovo-ideacentre-mini-01q8x10.dts @@ -1101,8 +1101,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j>; - vdda-pll-supply = <&vreg_l1j>; + vdda-phy-supply = <&vreg_l1j>; + vdda-pll-supply = <&vreg_l2j>; status = "okay"; }; @@ -1112,8 +1112,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j>; - vdda-pll-supply = <&vreg_l2d>; + vdda-phy-supply = <&vreg_l2d>; + vdda-pll-supply = <&vreg_l2j>; /delete-property/ mode-switch; /delete-property/ orientation-switch; @@ -1138,8 +1138,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j>; - vdda-pll-supply = <&vreg_l2d>; + vdda-phy-supply = <&vreg_l2d>; + vdda-pll-supply = <&vreg_l2j>; /delete-property/ mode-switch; /delete-property/ orientation-switch; @@ -1186,15 +1186,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e>; - vdda-pll-supply = <&vreg_l3c>; + vdda-phy-supply = <&vreg_l3c>; + vdda-pll-supply = <&vreg_l3e>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e>; - vdda-pll-supply = <&vreg_l3c>; + vdda-phy-supply = <&vreg_l3c>; + vdda-pll-supply = <&vreg_l3e>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/hamoa.dtsi b/arch/arm64/boot/dts/qcom/hamoa.dtsi index 4ba751a65142ba..fd86f4761eab9a 100644 --- a/arch/arm64/boot/dts/qcom/hamoa.dtsi +++ b/arch/arm64/boot/dts/qcom/hamoa.dtsi @@ -2868,7 +2868,7 @@ reg = <0 0x00fd3000 0 0x154>; #phy-cells = <0>; - clocks = <&tcsr TCSR_USB2_1_CLKREF_EN>; + clocks = <&rpmhcc RPMH_CXO_CLK>; clock-names = "ref"; resets = <&gcc GCC_QUSB2PHY_PRIM_BCR>; @@ -3010,7 +3010,7 @@ reg = <0 0x00fde000 0 0x154>; #phy-cells = <0>; - clocks = <&tcsr TCSR_USB2_1_CLKREF_EN>; + clocks = <&tcsr TCSR_USB2_2_CLKREF_EN>; clock-names = "ref"; resets = <&gcc GCC_QUSB2PHY_TERT_BCR>; diff --git a/arch/arm64/boot/dts/qcom/ipq5018.dtsi b/arch/arm64/boot/dts/qcom/ipq5018.dtsi index 6f8004a22a1ffd..52fc2d266b44b5 100644 --- a/arch/arm64/boot/dts/qcom/ipq5018.dtsi +++ b/arch/arm64/boot/dts/qcom/ipq5018.dtsi @@ -229,6 +229,9 @@ compatible = "ethernet-phy-id004d.d0c0"; reg = <7>; + clocks = <&gcc GCC_GEPHY_RX_CLK>, + <&gcc GCC_GEPHY_TX_CLK>; + clock-names = "rx", "tx"; resets = <&gcc GCC_GEPHY_MISC_ARES>; }; }; @@ -256,7 +259,7 @@ "sys"; #clock-cells = <1>; assigned-clocks = <&cmn_pll IPQ5018_CMN_PLL_CLK>; - assigned-clock-rates-u64 = /bits/ 64 <9600000000>; + assigned-clock-rates-u64 = /bits/ 64 <4800000000>; }; qfprom: qfprom@a0000 { diff --git a/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts b/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts index f9b5b5718b9046..2c1ae40dc7ec92 100644 --- a/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts +++ b/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts @@ -1343,8 +1343,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l1d_1p2>; - vdda-pll-supply = <&vreg_l4f_0p8>; + vdda-phy-supply = <&vreg_l4f_0p8>; + vdda-pll-supply = <&vreg_l1d_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts b/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts index 55d02219ef4e90..28779698221d27 100644 --- a/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts +++ b/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts @@ -838,8 +838,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l1d_1p2>; - vdda-pll-supply = <&vreg_l4f_0p8>; + vdda-phy-supply = <&vreg_l4f_0p8>; + vdda-pll-supply = <&vreg_l1d_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/kaanapali.dtsi b/arch/arm64/boot/dts/qcom/kaanapali.dtsi index 7aa9653bd456ed..011fb00beb0eac 100644 --- a/arch/arm64/boot/dts/qcom/kaanapali.dtsi +++ b/arch/arm64/boot/dts/qcom/kaanapali.dtsi @@ -2310,8 +2310,8 @@ operating-points-v2 = <&pcie0_opp_table>; - iommu-map = <0 &apps_smmu 0x1400 0x1>, - <0x100 &apps_smmu 0x1401 0x1>; + iommu-map = <0 &apps_smmu 0x1400 0x0 0x1>, + <0x100 &apps_smmu 0x1401 0x0 0x1>; interrupt-map = <0 0 0 1 &intc 0 0 GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>, <0 0 0 2 &intc 0 0 GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>, diff --git a/arch/arm64/boot/dts/qcom/kodiak.dtsi b/arch/arm64/boot/dts/qcom/kodiak.dtsi index ba907760fa6d62..48639568f15e4f 100644 --- a/arch/arm64/boot/dts/qcom/kodiak.dtsi +++ b/arch/arm64/boot/dts/qcom/kodiak.dtsi @@ -2281,8 +2281,8 @@ "aggre0", "aggre1"; - iommu-map = <0x0 &apps_smmu 0x1c00 0x1>, - <0x100 &apps_smmu 0x1c01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c00 0x0 0x1>, + <0x100 &apps_smmu 0x1c01 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -2427,8 +2427,8 @@ dma-coherent; - iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, - <0x100 &apps_smmu 0x1c81 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c80 0x0 0x1>, + <0x100 &apps_smmu 0x1c81 0x0 0x1>; status = "disabled"; diff --git a/arch/arm64/boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso b/arch/arm64/boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso index 44bd9b1a17652b..22c975628ed791 100644 --- a/arch/arm64/boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso +++ b/arch/arm64/boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso @@ -204,15 +204,15 @@ }; &pcie0 { - iommu-map = <0x0 &pcie_smmu 0x0 0x1>, - <0x100 &pcie_smmu 0x1 0x1>, - <0x208 &pcie_smmu 0x2 0x1>, - <0x210 &pcie_smmu 0x3 0x1>, - <0x218 &pcie_smmu 0x4 0x1>, - <0x300 &pcie_smmu 0x5 0x1>, - <0x400 &pcie_smmu 0x6 0x1>, - <0x500 &pcie_smmu 0x7 0x1>, - <0x501 &pcie_smmu 0x8 0x1>; + iommu-map = <0x0 &pcie_smmu 0x0 0x0 0x1>, + <0x100 &pcie_smmu 0x1 0x0 0x1>, + <0x208 &pcie_smmu 0x2 0x0 0x1>, + <0x210 &pcie_smmu 0x3 0x0 0x1>, + <0x218 &pcie_smmu 0x4 0x0 0x1>, + <0x300 &pcie_smmu 0x5 0x0 0x1>, + <0x400 &pcie_smmu 0x6 0x0 0x1>, + <0x500 &pcie_smmu 0x7 0x0 0x1>, + <0x501 &pcie_smmu 0x8 0x0 0x1>; }; &pcieport0 { diff --git a/arch/arm64/boot/dts/qcom/lemans-evk.dts b/arch/arm64/boot/dts/qcom/lemans-evk.dts index 34dfc8d22b6a58..b835fce8b336fc 100644 --- a/arch/arm64/boot/dts/qcom/lemans-evk.dts +++ b/arch/arm64/boot/dts/qcom/lemans-evk.dts @@ -1008,8 +1008,8 @@ }; &usb_0_qmpphy { - vdda-phy-supply = <&vreg_l1c>; - vdda-pll-supply = <&vreg_l7a>; + vdda-phy-supply = <&vreg_l7a>; + vdda-pll-supply = <&vreg_l1c>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi b/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi index 40f88498999bd8..b2195fc0309570 100644 --- a/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi +++ b/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi @@ -1103,8 +1103,8 @@ }; &usb_0_qmpphy { - vdda-phy-supply = <&vreg_l1c>; - vdda-pll-supply = <&vreg_l7a>; + vdda-phy-supply = <&vreg_l7a>; + vdda-pll-supply = <&vreg_l1c>; status = "okay"; }; @@ -1126,8 +1126,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l1c>; - vdda-pll-supply = <&vreg_l7a>; + vdda-phy-supply = <&vreg_l7a>; + vdda-pll-supply = <&vreg_l1c>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/lemans.dtsi b/arch/arm64/boot/dts/qcom/lemans.dtsi index 353a6e6fd3acb2..47d26dd832f7db 100644 --- a/arch/arm64/boot/dts/qcom/lemans.dtsi +++ b/arch/arm64/boot/dts/qcom/lemans.dtsi @@ -2760,8 +2760,8 @@ <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_0 0>; interconnect-names = "pcie-mem", "cpu-pcie"; - iommu-map = <0x0 &pcie_smmu 0x0000 0x1>, - <0x100 &pcie_smmu 0x0001 0x1>; + iommu-map = <0x0 &pcie_smmu 0x0000 0x0 0x1>, + <0x100 &pcie_smmu 0x0001 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>, <&gcc GCC_PCIE_0_LINK_DOWN_BCR>; @@ -2933,8 +2933,8 @@ <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_1 0>; interconnect-names = "pcie-mem", "cpu-pcie"; - iommu-map = <0x0 &pcie_smmu 0x0080 0x1>, - <0x100 &pcie_smmu 0x0081 0x1>; + iommu-map = <0x0 &pcie_smmu 0x0080 0x0 0x1>, + <0x100 &pcie_smmu 0x0081 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>, <&gcc GCC_PCIE_1_LINK_DOWN_BCR>; diff --git a/arch/arm64/boot/dts/qcom/milos.dtsi b/arch/arm64/boot/dts/qcom/milos.dtsi index 8c92329889538f..262fa88012562c 100644 --- a/arch/arm64/boot/dts/qcom/milos.dtsi +++ b/arch/arm64/boot/dts/qcom/milos.dtsi @@ -1724,6 +1724,8 @@ iommus = <&apps_smmu 0x540 0>; + resets = <&gcc GCC_SDCC2_BCR>; + bus-width = <4>; qcom,dll-config = <0x0007442c>; diff --git a/arch/arm64/boot/dts/qcom/monaco-evk-ifp-mezzanine.dtso b/arch/arm64/boot/dts/qcom/monaco-evk-ifp-mezzanine.dtso index e6beb4393430b1..a6e57ec8a1e14c 100644 --- a/arch/arm64/boot/dts/qcom/monaco-evk-ifp-mezzanine.dtso +++ b/arch/arm64/boot/dts/qcom/monaco-evk-ifp-mezzanine.dtso @@ -50,15 +50,15 @@ }; &pcie0 { - iommu-map = <0x0 &pcie_smmu 0x0 0x1>, - <0x100 &pcie_smmu 0x1 0x1>, - <0x208 &pcie_smmu 0x2 0x1>, - <0x210 &pcie_smmu 0x3 0x1>, - <0x218 &pcie_smmu 0x4 0x1>, - <0x300 &pcie_smmu 0x5 0x1>, - <0x400 &pcie_smmu 0x6 0x1>, - <0x500 &pcie_smmu 0x7 0x1>, - <0x501 &pcie_smmu 0x8 0x1>; + iommu-map = <0x0 &pcie_smmu 0x0 0x0 0x1>, + <0x100 &pcie_smmu 0x1 0x0 0x1>, + <0x208 &pcie_smmu 0x2 0x0 0x1>, + <0x210 &pcie_smmu 0x3 0x0 0x1>, + <0x218 &pcie_smmu 0x4 0x0 0x1>, + <0x300 &pcie_smmu 0x5 0x0 0x1>, + <0x400 &pcie_smmu 0x6 0x0 0x1>, + <0x500 &pcie_smmu 0x7 0x0 0x1>, + <0x501 &pcie_smmu 0x8 0x0 0x1>; }; &pcieport0 { diff --git a/arch/arm64/boot/dts/qcom/monaco-monza-som.dtsi b/arch/arm64/boot/dts/qcom/monaco-monza-som.dtsi index 9b5ed55939b866..36af1ad2105d5b 100644 --- a/arch/arm64/boot/dts/qcom/monaco-monza-som.dtsi +++ b/arch/arm64/boot/dts/qcom/monaco-monza-som.dtsi @@ -196,14 +196,14 @@ /* PCIe0 Gen4 x2 */ &pcie0 { - iommu-map = <0x0 &pcie_smmu 0x0000 0x1>, - <0x100 &pcie_smmu 0x0001 0x1>, - <0x200 &pcie_smmu 0x0007 0x1>, - <0x208 &pcie_smmu 0x0002 0x1>, - <0x210 &pcie_smmu 0x0003 0x1>, - <0x218 &pcie_smmu 0x0004 0x1>, - <0x300 &pcie_smmu 0x0005 0x1>, - <0x400 &pcie_smmu 0x0006 0x1>; + iommu-map = <0x0 &pcie_smmu 0x0000 0x0 0x1>, + <0x100 &pcie_smmu 0x0001 0x0 0x1>, + <0x200 &pcie_smmu 0x0007 0x0 0x1>, + <0x208 &pcie_smmu 0x0002 0x0 0x1>, + <0x210 &pcie_smmu 0x0003 0x0 0x1>, + <0x218 &pcie_smmu 0x0004 0x0 0x1>, + <0x300 &pcie_smmu 0x0005 0x0 0x1>, + <0x400 &pcie_smmu 0x0006 0x0 0x1>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/monaco.dtsi b/arch/arm64/boot/dts/qcom/monaco.dtsi index a350a8ba48fa39..61ef690f5fdf74 100644 --- a/arch/arm64/boot/dts/qcom/monaco.dtsi +++ b/arch/arm64/boot/dts/qcom/monaco.dtsi @@ -2349,8 +2349,8 @@ interconnect-names = "pcie-mem", "cpu-pcie"; - iommu-map = <0x0 &pcie_smmu 0x0000 0x1>, - <0x100 &pcie_smmu 0x0001 0x1>; + iommu-map = <0x0 &pcie_smmu 0x0000 0x0 0x1>, + <0x100 &pcie_smmu 0x0001 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>, <&gcc GCC_PCIE_0_LINK_DOWN_BCR>; @@ -2526,8 +2526,8 @@ &config_noc SLAVE_PCIE_1 QCOM_ICC_TAG_ACTIVE_ONLY>; interconnect-names = "pcie-mem", "cpu-pcie"; - iommu-map = <0x0 &pcie_smmu 0x0080 0x1>, - <0x100 &pcie_smmu 0x0081 0x1>; + iommu-map = <0x0 &pcie_smmu 0x0080 0x0 0x1>, + <0x100 &pcie_smmu 0x0081 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>, <&gcc GCC_PCIE_1_LINK_DOWN_BCR>; @@ -3017,14 +3017,6 @@ #address-cells = <1>; #size-cells = <0>; - port@0 { - reg = <0>; - - swao_rep_out0: endpoint { - remote-endpoint = <&qdss_rep_in>; - }; - }; - port@1 { reg = <1>; @@ -3734,6 +3726,14 @@ #address-cells = <1>; #size-cells = <0>; + port@0 { + reg = <0>; + + swao_rep_out0: endpoint { + remote-endpoint = <&qdss_rep_in>; + }; + }; + port@1 { reg = <1>; diff --git a/arch/arm64/boot/dts/qcom/msm8976-longcheer-l9360.dts b/arch/arm64/boot/dts/qcom/msm8976-longcheer-l9360.dts index 18832a3b9a1c37..57f549f06f73cb 100644 --- a/arch/arm64/boot/dts/qcom/msm8976-longcheer-l9360.dts +++ b/arch/arm64/boot/dts/qcom/msm8976-longcheer-l9360.dts @@ -455,14 +455,14 @@ bias-disable; }; - ts_int_default: ts-int-state { + ts_int_default: ts-int-default-state { pins = "gpio65"; function = "gpio"; drive-strength = <2>; bias-pull-down; }; - ts_int_sleep: ts-int-state { + ts_int_sleep: ts-int-sleep-state { pins = "gpio65"; function = "gpio"; drive-strength = <2>; diff --git a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts index fd3a2121465b00..ca22e2f9d20a20 100644 --- a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts +++ b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts @@ -39,7 +39,7 @@ haptics: drv2604@5a { compatible = "ti,drv2604"; reg = <0x5a>; - enable-gpio = <&tlmm 93 0x00>; + enable-gpios = <&tlmm 93 GPIO_ACTIVE_HIGH>; mode = ; library-sel = ; pinctrl-names = "default","sleep"; diff --git a/arch/arm64/boot/dts/qcom/msm8998-sony-xperia-yoshino.dtsi b/arch/arm64/boot/dts/qcom/msm8998-sony-xperia-yoshino.dtsi index 3650f2501886bd..04d4741cdb5f02 100644 --- a/arch/arm64/boot/dts/qcom/msm8998-sony-xperia-yoshino.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8998-sony-xperia-yoshino.dtsi @@ -229,10 +229,6 @@ }; }; -&blsp1_i2c5_sleep { - bias-disable; -}; - &blsp1_uart3 { status = "okay"; diff --git a/arch/arm64/boot/dts/qcom/msm8998-xiaomi-sagit.dts b/arch/arm64/boot/dts/qcom/msm8998-xiaomi-sagit.dts index 30222f6608da75..69528771eda259 100644 --- a/arch/arm64/boot/dts/qcom/msm8998-xiaomi-sagit.dts +++ b/arch/arm64/boot/dts/qcom/msm8998-xiaomi-sagit.dts @@ -217,11 +217,6 @@ }; }; -&blsp1_i2c5_sleep { - /delete-property/ bias-pull-up; - bias-disable; -}; - &blsp1_uart3 { status = "okay"; diff --git a/arch/arm64/boot/dts/qcom/msm8998.dtsi b/arch/arm64/boot/dts/qcom/msm8998.dtsi index 34770601163729..5038d0009c1a90 100644 --- a/arch/arm64/boot/dts/qcom/msm8998.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8998.dtsi @@ -1183,7 +1183,7 @@ pins = "gpio2", "gpio3"; function = "blsp_i2c1"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp1_i2c2_default: blsp1-i2c2-default-state { @@ -1197,7 +1197,7 @@ pins = "gpio32", "gpio33"; function = "blsp_i2c2"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp1_i2c3_default: blsp1-i2c3-default-state { @@ -1211,7 +1211,7 @@ pins = "gpio47", "gpio48"; function = "blsp_i2c3"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp1_i2c4_default: blsp1-i2c4-default-state { @@ -1225,7 +1225,7 @@ pins = "gpio10", "gpio11"; function = "blsp_i2c4"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp1_i2c5_default: blsp1-i2c5-default-state { @@ -1239,7 +1239,7 @@ pins = "gpio87", "gpio88"; function = "blsp_i2c5"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp1_i2c6_default: blsp1-i2c6-default-state { @@ -1253,7 +1253,7 @@ pins = "gpio43", "gpio44"; function = "blsp_i2c6"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp1_spi_b_default: blsp1-spi-b-default-state { @@ -1318,7 +1318,7 @@ pins = "gpio55", "gpio56"; function = "blsp_i2c7"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp2_i2c2_default: blsp2-i2c2-default-state { @@ -1332,7 +1332,7 @@ pins = "gpio6", "gpio7"; function = "blsp_i2c8"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp2_i2c3_default: blsp2-i2c3-default-state { @@ -1346,7 +1346,7 @@ pins = "gpio51", "gpio52"; function = "blsp_i2c9"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp2_i2c4_default: blsp2-i2c4-default-state { @@ -1360,7 +1360,7 @@ pins = "gpio67", "gpio68"; function = "blsp_i2c10"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp2_i2c5_default: blsp2-i2c5-default-state { @@ -1374,7 +1374,7 @@ pins = "gpio60", "gpio61"; function = "blsp_i2c11"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp2_i2c6_default: blsp2-i2c6-default-state { @@ -1388,7 +1388,7 @@ pins = "gpio83", "gpio84"; function = "blsp_i2c12"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp2_spi1_default: blsp2-spi1-default-state { diff --git a/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi b/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi index 394e65518ac503..50ea8a148ada03 100644 --- a/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi +++ b/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi @@ -583,8 +583,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -604,8 +604,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -625,8 +625,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -663,15 +663,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts b/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts index 04cb9230d29fd1..92a7d80b7fee62 100644 --- a/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts +++ b/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts @@ -1502,8 +1502,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l6b>; - vdda-pll-supply = <&vreg_l1b>; + vdda-phy-supply = <&vreg_l1b>; + vdda-pll-supply = <&vreg_l6b>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/qcm6490-idp.dts b/arch/arm64/boot/dts/qcom/qcm6490-idp.dts index bdc02260f902bd..58cce89a0c1ce8 100644 --- a/arch/arm64/boot/dts/qcom/qcm6490-idp.dts +++ b/arch/arm64/boot/dts/qcom/qcm6490-idp.dts @@ -1141,8 +1141,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p912>; + vdda-phy-supply = <&vreg_l1b_0p912>; + vdda-pll-supply = <&vreg_l6b_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts b/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts index bf18c485208137..17804de03e1776 100644 --- a/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts +++ b/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts @@ -839,8 +839,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p912>; + vdda-phy-supply = <&vreg_l1b_0p912>; + vdda-pll-supply = <&vreg_l6b_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/qcm6490-shift-otter.dts b/arch/arm64/boot/dts/qcom/qcm6490-shift-otter.dts index 797f37596bf197..cdaacc09f56c1d 100644 --- a/arch/arm64/boot/dts/qcom/qcm6490-shift-otter.dts +++ b/arch/arm64/boot/dts/qcom/qcm6490-shift-otter.dts @@ -976,8 +976,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l6b>; - vdda-pll-supply = <&vreg_l1b>; + vdda-phy-supply = <&vreg_l1b>; + vdda-pll-supply = <&vreg_l6b>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/qcs404.dtsi b/arch/arm64/boot/dts/qcom/qcs404.dtsi index 4328c1dda898c2..736accfc34f904 100644 --- a/arch/arm64/boot/dts/qcom/qcs404.dtsi +++ b/arch/arm64/boot/dts/qcom/qcs404.dtsi @@ -677,11 +677,11 @@ assigned-clock-rates = <19200000>, <200000000>; interrupts = , - , - ; + , + ; interrupt-names = "pwr_event", - "hs_phy_irq", - "qusb2_phy"; + "qusb2_phy", + "hs_phy_irq"; status = "disabled"; @@ -716,11 +716,11 @@ assigned-clock-rates = <19200000>, <133333333>; interrupts = , - , - ; + , + ; interrupt-names = "pwr_event", - "hs_phy_irq", - "qusb2_phy"; + "qusb2_phy", + "hs_phy_irq"; status = "disabled"; diff --git a/arch/arm64/boot/dts/qcom/qcs6490-radxa-dragon-q6a.dts b/arch/arm64/boot/dts/qcom/qcs6490-radxa-dragon-q6a.dts index bb5a42b038f19b..696fef50f577b3 100644 --- a/arch/arm64/boot/dts/qcom/qcs6490-radxa-dragon-q6a.dts +++ b/arch/arm64/boot/dts/qcom/qcs6490-radxa-dragon-q6a.dts @@ -546,15 +546,15 @@ pinctrl-names = "default"; /* Support for QPS615 PCIe switch */ - iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, - <0x100 &apps_smmu 0x1c81 0x1>, - <0x208 &apps_smmu 0x1c84 0x1>, - <0x210 &apps_smmu 0x1c85 0x1>, - <0x218 &apps_smmu 0x1c86 0x1>, - <0x300 &apps_smmu 0x1c87 0x1>, - <0x400 &apps_smmu 0x1c88 0x1>, - <0x500 &apps_smmu 0x1c89 0x1>, - <0x501 &apps_smmu 0x1c90 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c80 0x0 0x1>, + <0x100 &apps_smmu 0x1c81 0x0 0x1>, + <0x208 &apps_smmu 0x1c84 0x0 0x1>, + <0x210 &apps_smmu 0x1c85 0x0 0x1>, + <0x218 &apps_smmu 0x1c86 0x0 0x1>, + <0x300 &apps_smmu 0x1c87 0x0 0x1>, + <0x400 &apps_smmu 0x1c88 0x0 0x1>, + <0x500 &apps_smmu 0x1c89 0x0 0x1>, + <0x501 &apps_smmu 0x1c90 0x0 0x1>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso index 83908db335afa1..14c64439ae1e6c 100644 --- a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso +++ b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso @@ -54,15 +54,15 @@ pinctrl-0 = <&pcie0_reset_n>, <&pcie0_wake_n>, <&pcie0_clkreq_n>; pinctrl-names = "default"; - iommu-map = <0x0 &apps_smmu 0x1c00 0x1>, - <0x100 &apps_smmu 0x1c01 0x1>, - <0x208 &apps_smmu 0x1c04 0x1>, - <0x210 &apps_smmu 0x1c05 0x1>, - <0x218 &apps_smmu 0x1c06 0x1>, - <0x300 &apps_smmu 0x1c07 0x1>, - <0x400 &apps_smmu 0x1c08 0x1>, - <0x500 &apps_smmu 0x1c09 0x1>, - <0x501 &apps_smmu 0x1c10 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c00 0x0 0x1>, + <0x100 &apps_smmu 0x1c01 0x0 0x1>, + <0x208 &apps_smmu 0x1c04 0x0 0x1>, + <0x210 &apps_smmu 0x1c05 0x0 0x1>, + <0x218 &apps_smmu 0x1c06 0x0 0x1>, + <0x300 &apps_smmu 0x1c07 0x0 0x1>, + <0x400 &apps_smmu 0x1c08 0x0 0x1>, + <0x500 &apps_smmu 0x1c09 0x0 0x1>, + <0x501 &apps_smmu 0x1c10 0x0 0x1>; status = "okay"; }; @@ -151,22 +151,22 @@ }; &pcie1 { - iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, - <0x100 &apps_smmu 0x1c81 0x1>, - <0x208 &apps_smmu 0x1c84 0x1>, - <0x210 &apps_smmu 0x1c85 0x1>, - <0x218 &apps_smmu 0x1c86 0x1>, - <0x300 &apps_smmu 0x1c87 0x1>, - <0x408 &apps_smmu 0x1c90 0x1>, - <0x410 &apps_smmu 0x1c91 0x1>, - <0x418 &apps_smmu 0x1c92 0x1>, - <0x500 &apps_smmu 0x1c93 0x1>, - <0x600 &apps_smmu 0x1c94 0x1>, - <0x700 &apps_smmu 0x1c95 0x1>, - <0x701 &apps_smmu 0x1c96 0x1>, - <0x800 &apps_smmu 0x1c97 0x1>, - <0x900 &apps_smmu 0x1c98 0x1>, - <0x901 &apps_smmu 0x1c99 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c80 0x0 0x1>, + <0x100 &apps_smmu 0x1c81 0x0 0x1>, + <0x208 &apps_smmu 0x1c84 0x0 0x1>, + <0x210 &apps_smmu 0x1c85 0x0 0x1>, + <0x218 &apps_smmu 0x1c86 0x0 0x1>, + <0x300 &apps_smmu 0x1c87 0x0 0x1>, + <0x408 &apps_smmu 0x1c90 0x0 0x1>, + <0x410 &apps_smmu 0x1c91 0x0 0x1>, + <0x418 &apps_smmu 0x1c92 0x0 0x1>, + <0x500 &apps_smmu 0x1c93 0x0 0x1>, + <0x600 &apps_smmu 0x1c94 0x0 0x1>, + <0x700 &apps_smmu 0x1c95 0x0 0x1>, + <0x701 &apps_smmu 0x1c96 0x0 0x1>, + <0x800 &apps_smmu 0x1c97 0x0 0x1>, + <0x900 &apps_smmu 0x1c98 0x0 0x1>, + <0x901 &apps_smmu 0x1c99 0x0 0x1>; }; &pcie1_switch0_dsp1 { diff --git a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts index 37a3b51323ce57..eeb422586896f4 100644 --- a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts +++ b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts @@ -846,15 +846,15 @@ pinctrl-0 = <&pcie1_reset_n>, <&pcie1_wake_n>, <&pcie1_clkreq_n>; pinctrl-names = "default"; - iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, - <0x100 &apps_smmu 0x1c81 0x1>, - <0x208 &apps_smmu 0x1c84 0x1>, - <0x210 &apps_smmu 0x1c85 0x1>, - <0x218 &apps_smmu 0x1c86 0x1>, - <0x300 &apps_smmu 0x1c87 0x1>, - <0x400 &apps_smmu 0x1c88 0x1>, - <0x500 &apps_smmu 0x1c89 0x1>, - <0x501 &apps_smmu 0x1c90 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c80 0x0 0x1>, + <0x100 &apps_smmu 0x1c81 0x0 0x1>, + <0x208 &apps_smmu 0x1c84 0x0 0x1>, + <0x210 &apps_smmu 0x1c85 0x0 0x1>, + <0x218 &apps_smmu 0x1c86 0x0 0x1>, + <0x300 &apps_smmu 0x1c87 0x0 0x1>, + <0x400 &apps_smmu 0x1c88 0x0 0x1>, + <0x500 &apps_smmu 0x1c89 0x0 0x1>, + <0x501 &apps_smmu 0x1c90 0x0 0x1>; status = "okay"; }; @@ -1419,8 +1419,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p912>; + vdda-phy-supply = <&vreg_l1b_0p912>; + vdda-pll-supply = <&vreg_l6b_1p2>; orientation-switch; diff --git a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts index a5ad796cb65d0f..1e773f9300f712 100644 --- a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts +++ b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts @@ -711,15 +711,15 @@ <&pcie1_wake_n>; pinctrl-names = "default"; - iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, - <0x100 &apps_smmu 0x1c81 0x1>, - <0x208 &apps_smmu 0x1c84 0x1>, - <0x210 &apps_smmu 0x1c85 0x1>, - <0x218 &apps_smmu 0x1c86 0x1>, - <0x300 &apps_smmu 0x1c87 0x1>, - <0x400 &apps_smmu 0x1c88 0x1>, - <0x500 &apps_smmu 0x1c89 0x1>, - <0x501 &apps_smmu 0x1c90 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c80 0x0 0x1>, + <0x100 &apps_smmu 0x1c81 0x0 0x1>, + <0x208 &apps_smmu 0x1c84 0x0 0x1>, + <0x210 &apps_smmu 0x1c85 0x0 0x1>, + <0x218 &apps_smmu 0x1c86 0x0 0x1>, + <0x300 &apps_smmu 0x1c87 0x0 0x1>, + <0x400 &apps_smmu 0x1c88 0x0 0x1>, + <0x500 &apps_smmu 0x1c89 0x0 0x1>, + <0x501 &apps_smmu 0x1c90 0x0 0x1>; status = "okay"; }; @@ -1059,8 +1059,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p912>; + vdda-phy-supply = <&vreg_l1b_0p912>; + vdda-pll-supply = <&vreg_l6b_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts index f47efca42d48d8..7336118ae7e349 100644 --- a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts +++ b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts @@ -1103,8 +1103,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p912>; + vdda-phy-supply = <&vreg_l1b_0p912>; + vdda-pll-supply = <&vreg_l6b_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/qcs8550-aim300.dtsi b/arch/arm64/boot/dts/qcom/qcs8550-aim300.dtsi index e6ebb643203b62..ae301d78e0710a 100644 --- a/arch/arm64/boot/dts/qcom/qcs8550-aim300.dtsi +++ b/arch/arm64/boot/dts/qcom/qcs8550-aim300.dtsi @@ -412,8 +412,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3f_0p88>; + vdda-phy-supply = <&vreg_l3f_0p88>; + vdda-pll-supply = <&vreg_l3e_1p2>; }; &xo_board { diff --git a/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts b/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts index 690b484352edf8..d6af679abe1a34 100644 --- a/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts +++ b/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts @@ -1424,8 +1424,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l9a_1p2>; - vdda-pll-supply = <&vreg_l18a_0p92>; + vdda-phy-supply = <&vreg_l18a_0p92>; + vdda-pll-supply = <&vreg_l9a_1p2>; }; &usb_1_qmpphy_out { @@ -1451,8 +1451,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l9a_1p2>; - vdda-pll-supply = <&vreg_l18a_0p92>; + vdda-phy-supply = <&vreg_l18a_0p92>; + vdda-pll-supply = <&vreg_l9a_1p2>; }; &vamacro { diff --git a/arch/arm64/boot/dts/qcom/sa8155p-adp.dts b/arch/arm64/boot/dts/qcom/sa8155p-adp.dts index 388d5ecee94978..3e22ee8b509246 100644 --- a/arch/arm64/boot/dts/qcom/sa8155p-adp.dts +++ b/arch/arm64/boot/dts/qcom/sa8155p-adp.dts @@ -428,8 +428,8 @@ &ufs_mem_phy { status = "okay"; - vdda-phy-supply = <&vreg_l8c_1p2>; - vdda-pll-supply = <&vreg_l5a_0p88>; + vdda-phy-supply = <&vreg_l5a_0p88>; + vdda-pll-supply = <&vreg_l8c_1p2>; }; &usb_1 { @@ -474,8 +474,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l8c_1p2>; - vdda-pll-supply = <&vdda_usb_ss_dp_core_1>; + vdda-phy-supply = <&vdda_usb_ss_dp_core_1>; + vdda-pll-supply = <&vreg_l8c_1p2>; }; &pcie0 { diff --git a/arch/arm64/boot/dts/qcom/sa8295p-adp.dts b/arch/arm64/boot/dts/qcom/sa8295p-adp.dts index d28d6916242791..7178ae087038ce 100644 --- a/arch/arm64/boot/dts/qcom/sa8295p-adp.dts +++ b/arch/arm64/boot/dts/qcom/sa8295p-adp.dts @@ -604,8 +604,8 @@ }; &usb_0_qmpphy { - vdda-phy-supply = <&vreg_l3a>; - vdda-pll-supply = <&vreg_l5a>; + vdda-phy-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l3a>; status = "okay"; }; @@ -628,8 +628,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l4c>; - vdda-pll-supply = <&vreg_l1c>; + vdda-phy-supply = <&vreg_l1c>; + vdda-pll-supply = <&vreg_l4c>; status = "okay"; }; @@ -671,15 +671,15 @@ }; &usb_2_qmpphy0 { - vdda-phy-supply = <&vreg_l3a>; - vdda-pll-supply = <&vreg_l5a>; + vdda-phy-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l3a>; status = "okay"; }; &usb_2_qmpphy1 { - vdda-phy-supply = <&vreg_l3a>; - vdda-pll-supply = <&vreg_l5a>; + vdda-phy-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l3a>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sa8540p-ride.dts b/arch/arm64/boot/dts/qcom/sa8540p-ride.dts index 44177e9b64b52b..df40c44c447e76 100644 --- a/arch/arm64/boot/dts/qcom/sa8540p-ride.dts +++ b/arch/arm64/boot/dts/qcom/sa8540p-ride.dts @@ -478,8 +478,8 @@ }; &usb_0_qmpphy { - vdda-phy-supply = <&vreg_l3a>; - vdda-pll-supply = <&vreg_l5a>; + vdda-phy-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l3a>; status = "okay"; }; @@ -493,8 +493,8 @@ }; &usb_2_qmpphy0 { - vdda-phy-supply = <&vreg_l3a>; - vdda-pll-supply = <&vreg_l5a>; + vdda-phy-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l3a>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sar2130p-qar2130p.dts b/arch/arm64/boot/dts/qcom/sar2130p-qar2130p.dts index 74778a5b19ba6d..80428c0b3b4e68 100644 --- a/arch/arm64/boot/dts/qcom/sar2130p-qar2130p.dts +++ b/arch/arm64/boot/dts/qcom/sar2130p-qar2130p.dts @@ -551,8 +551,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3a_1p2>; - vdda-pll-supply = <&vreg_l1a_0p91>; + vdda-phy-supply = <&vreg_l1a_0p91>; + vdda-pll-supply = <&vreg_l3a_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sar2130p.dtsi b/arch/arm64/boot/dts/qcom/sar2130p.dtsi index 3c9529bb2f76f5..99002ddaf6f638 100644 --- a/arch/arm64/boot/dts/qcom/sar2130p.dtsi +++ b/arch/arm64/boot/dts/qcom/sar2130p.dtsi @@ -1329,8 +1329,8 @@ &config_noc SLAVE_PCIE_0 QCOM_ICC_TAG_ALWAYS>; interconnect-names = "pcie-mem", "cpu-pcie"; - iommu-map = <0x0 &apps_smmu 0x1c00 0x1>, - <0x100 &apps_smmu 0x1c01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c00 0x0 0x1>, + <0x100 &apps_smmu 0x1c01 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -1455,8 +1455,8 @@ &config_noc SLAVE_PCIE_1 QCOM_ICC_TAG_ALWAYS>; interconnect-names = "pcie-mem", "cpu-pcie"; - iommu-map = <0x0 &apps_smmu 0x1e00 0x1>, - <0x100 &apps_smmu 0x1e01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1e00 0x0 0x1>, + <0x100 &apps_smmu 0x1e01 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>, <&gcc GCC_PCIE_1_LINK_DOWN_BCR>; diff --git a/arch/arm64/boot/dts/qcom/sc7180-acer-aspire1.dts b/arch/arm64/boot/dts/qcom/sc7180-acer-aspire1.dts index 1514da63626982..d9c70897b6f1c0 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-acer-aspire1.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-acer-aspire1.dts @@ -631,8 +631,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l4a_0p8>; + vdda-phy-supply = <&vreg_l4a_0p8>; + vdda-pll-supply = <&vreg_l3c_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sc7180-ecs-liva-qc710.dts b/arch/arm64/boot/dts/qcom/sc7180-ecs-liva-qc710.dts index b84da371581b9f..3100fab18787a8 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-ecs-liva-qc710.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-ecs-liva-qc710.dts @@ -563,8 +563,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l4a_0p8>; + vdda-phy-supply = <&vreg_l4a_0p8>; + vdda-pll-supply = <&vreg_l3c_1p2>; /delete-property/ mode-switch; /delete-property/ orientation-switch; diff --git a/arch/arm64/boot/dts/qcom/sc7180-idp.dts b/arch/arm64/boot/dts/qcom/sc7180-idp.dts index 0bce3eefca2e9b..acfb9937651d66 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-idp.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-idp.dts @@ -461,8 +461,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l4a_0p8>; + vdda-phy-supply = <&vreg_l4a_0p8>; + vdda-pll-supply = <&vreg_l3c_1p2>; }; &venus { diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi b/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi index b398f69917f0e1..0e0bc8543f714e 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi @@ -99,6 +99,11 @@ stdout-path = "serial0:115200n8"; }; + firmware { + #address-cells = <2>; + #size-cells = <2>; + }; + /* FIXED REGULATORS - parents above children */ /* This is the top level supply and variable voltage */ @@ -993,8 +998,8 @@ ap_spi_fp: &spi10 { &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vdda_usb_ss_dp_1p2>; - vdda-pll-supply = <&vdda_usb_ss_dp_core>; + vdda-phy-supply = <&vdda_usb_ss_dp_core>; + vdda-pll-supply = <&vdda_usb_ss_dp_1p2>; }; &venus { diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi b/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi index 5c5e4f1dd22173..58ea0532c0fbbd 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi @@ -25,6 +25,11 @@ stdout-path = "serial0:115200n8"; }; + firmware { + #address-cells = <2>; + #size-cells = <2>; + }; + /* * FIXED REGULATORS * diff --git a/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi b/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi index 8cac4ce9c85153..23ced0621c99e3 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi @@ -542,8 +542,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p8>; + vdda-phy-supply = <&vreg_l1b_0p8>; + vdda-pll-supply = <&vreg_l6b_1p2>; }; &uart7 { diff --git a/arch/arm64/boot/dts/qcom/sc7280-qcard.dtsi b/arch/arm64/boot/dts/qcom/sc7280-qcard.dtsi index 9ae9af40f4d212..264029ff3f903e 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-qcard.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7280-qcard.dtsi @@ -465,8 +465,8 @@ mos_bt_uart: &uart7 { }; &usb_1_qmpphy { - vdda-phy-supply = <&vdd_a_usbssdp_0_1p2>; - vdda-pll-supply = <&vdd_a_usbssdp_0_core>; + vdda-phy-supply = <&vdd_a_usbssdp_0_core>; + vdda-pll-supply = <&vdd_a_usbssdp_0_1p2>; }; &usb_2_hsphy { diff --git a/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts b/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts index d86a31ddede29d..565e5a9979b1ad 100644 --- a/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts +++ b/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts @@ -26,6 +26,7 @@ compatible = "pwm-backlight"; pwms = <&pmc8180c_lpg 4 1000000>; enable-gpios = <&pmc8180c_gpios 8 GPIO_ACTIVE_HIGH>; + power-supply = <&vled_bl_pw>; pinctrl-0 = <&bl_pwm_default>; pinctrl-names = "default"; @@ -157,14 +158,39 @@ }; }; - vph_pwr: vph-pwr-regulator { + vled_bl_pw: regulator-vled-bl-pw { compatible = "regulator-fixed"; - regulator-name = "vph_pwr"; - regulator-min-microvolt = <3700000>; - regulator-max-microvolt = <3700000>; + + regulator-name = "VLED_BL_PW"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&pmc8180_2_gpios 1 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&bl_pwr_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_lcm_3v3: regulator-edp-3p3 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_LCM_3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&tlmm 130 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&lcm_3v3_en>; + pinctrl-names = "default"; + + regulator-boot-on; }; - vreg_s4a_1p8: pm8150-s4-regulator { + vreg_s4a_1p8: regulator-pm8150-s4 { compatible = "regulator-fixed"; regulator-name = "vreg_s4a_1p8"; @@ -177,6 +203,13 @@ vin-supply = <&vph_pwr>; }; + vph_pwr: regulator-vph-pwr { + compatible = "regulator-fixed"; + regulator-name = "vph_pwr"; + regulator-min-microvolt = <3700000>; + regulator-max-microvolt = <3700000>; + }; + usbprim-sbu-mux { compatible = "pericom,pi3usb102", "gpio-sbu-mux"; @@ -438,6 +471,7 @@ aux-bus { panel { compatible = "edp-panel"; + power-supply = <&vreg_lcm_3v3>; no-hpd; backlight = <&backlight>; @@ -472,6 +506,13 @@ status = "okay"; }; +&pmc8180_2_gpios { + bl_pwr_en: bl-pwr-en-state { + pins = "gpio1"; + function = "normal"; + }; +}; + &pmc8180_pwrkey { status = "okay"; }; @@ -570,15 +611,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l5e_0p88>; + vdda-phy-supply = <&vreg_l5e_0p88>; + vdda-pll-supply = <&vreg_l3c_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l5e_0p88>; + vdda-phy-supply = <&vreg_l5e_0p88>; + vdda-pll-supply = <&vreg_l3c_1p2>; status = "okay"; }; @@ -592,8 +633,8 @@ }; &usb_prim_qmpphy { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l5e_0p88>; + vdda-phy-supply = <&vreg_l5e_0p88>; + vdda-pll-supply = <&vreg_l3c_1p2>; orientation-switch; @@ -633,8 +674,8 @@ }; &usb_sec_qmpphy { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l5e_0p88>; + vdda-phy-supply = <&vreg_l5e_0p88>; + vdda-pll-supply = <&vreg_l3c_1p2>; orientation-switch; @@ -765,6 +806,12 @@ drive-strength = <2>; }; + lcm_3v3_en: lcm-3v3-en-state { + pins = "gpio130"; + function = "gpio"; + bias-disable; + }; + usbprim_sbu_default: usbprim-sbu-state { oe-n-pins { pins = "gpio152"; diff --git a/arch/arm64/boot/dts/qcom/sc8180x-primus.dts b/arch/arm64/boot/dts/qcom/sc8180x-primus.dts index aff398390eba77..c10a936b320a99 100644 --- a/arch/arm64/boot/dts/qcom/sc8180x-primus.dts +++ b/arch/arm64/boot/dts/qcom/sc8180x-primus.dts @@ -29,9 +29,10 @@ compatible = "pwm-backlight"; pwms = <&pmc8180c_lpg 4 1000000>; enable-gpios = <&pmc8180c_gpios 8 GPIO_ACTIVE_HIGH>; + power-supply = <&vled_bl_pw>; - pinctrl-names = "default"; pinctrl-0 = <&bl_pwm_default>; + pinctrl-names = "default"; }; chosen { @@ -167,7 +168,39 @@ }; }; - vreg_nvme_0p9: nvme-0p9-regulator { + vled_bl_pw: regulator-vled-bl-pw { + compatible = "regulator-fixed"; + + regulator-name = "VLED_BL_PW"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&pmc8180_2_gpios 1 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&bl_pwr_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_lcm_3v3: regulator-edp-3p3 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_LCM_3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&tlmm 130 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&lcm_3v3_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_nvme_0p9: regulator-nvme-0p9 { compatible = "regulator-fixed"; regulator-name = "vreg_nvme_0p9"; @@ -177,7 +210,7 @@ regulator-always-on; }; - vreg_nvme_3p3: nvme-3p3-regulator { + vreg_nvme_3p3: regulator-nvme-3p3 { compatible = "regulator-fixed"; regulator-name = "vreg_nvme_3p3"; @@ -190,7 +223,7 @@ regulator-always-on; }; - vdd_kb_tp_3v3: vdd-kb-tp-3v3-regulator { + vdd_kb_tp_3v3: regulator-vdd-kb-tp-3v3 { compatible = "regulator-fixed"; regulator-name = "vdd_kb_tp_3v3"; regulator-min-microvolt = <3300000>; @@ -205,7 +238,7 @@ pinctrl-0 = <&kb_tp_3v3_en_active_state>; }; - vph_pwr: vph-pwr-regulator { + vph_pwr: regulator-vph-pwr { compatible = "regulator-fixed"; regulator-name = "vph_pwr"; regulator-min-microvolt = <3700000>; @@ -540,6 +573,7 @@ aux-bus { panel { compatible = "edp-panel"; + power-supply = <&vreg_lcm_3v3>; backlight = <&backlight>; @@ -669,15 +703,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l5e_0p88>; + vdda-phy-supply = <&vreg_l5e_0p88>; + vdda-pll-supply = <&vreg_l3c_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l5e_0p88>; + vdda-phy-supply = <&vreg_l5e_0p88>; + vdda-pll-supply = <&vreg_l3c_1p2>; status = "okay"; }; @@ -691,8 +725,8 @@ }; &usb_prim_qmpphy { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l5e_0p88>; + vdda-phy-supply = <&vreg_l5e_0p88>; + vdda-pll-supply = <&vreg_l3c_1p2>; orientation-switch; @@ -728,8 +762,8 @@ }; &usb_sec_qmpphy { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l5e_0p88>; + vdda-phy-supply = <&vreg_l5e_0p88>; + vdda-pll-supply = <&vreg_l3c_1p2>; orientation-switch; @@ -769,6 +803,12 @@ }; /* PINCTRL */ +&pmc8180_2_gpios { + bl_pwr_en: bl-pwr-en-state { + pins = "gpio1"; + function = "normal"; + }; +}; &pmc8180c_gpios { bl_pwm_default: bl-pwm-default-state { @@ -950,4 +990,10 @@ bias-pull-up; }; }; + + lcm_3v3_en: lcm-3v3-en-state { + pins = "gpio130"; + function = "gpio"; + bias-disable; + }; }; diff --git a/arch/arm64/boot/dts/qcom/sc8180x.dtsi b/arch/arm64/boot/dts/qcom/sc8180x.dtsi index 45391768e24582..5d3f0c4d7b464a 100644 --- a/arch/arm64/boot/dts/qcom/sc8180x.dtsi +++ b/arch/arm64/boot/dts/qcom/sc8180x.dtsi @@ -1767,8 +1767,8 @@ assigned-clocks = <&gcc GCC_PCIE_0_AUX_CLK>; assigned-clock-rates = <19200000>; - iommu-map = <0x0 &apps_smmu 0x1d80 0x1>, - <0x100 &apps_smmu 0x1d81 0x1>; + iommu-map = <0x0 &apps_smmu 0x1d80 0x0 0x1>, + <0x100 &apps_smmu 0x1d81 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -1886,8 +1886,8 @@ assigned-clocks = <&gcc GCC_PCIE_3_AUX_CLK>; assigned-clock-rates = <19200000>; - iommu-map = <0x0 &apps_smmu 0x1e00 0x1>, - <0x100 &apps_smmu 0x1e01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1e00 0x0 0x1>, + <0x100 &apps_smmu 0x1e01 0x0 0x1>; resets = <&gcc GCC_PCIE_3_BCR>; reset-names = "pci"; @@ -2006,8 +2006,8 @@ assigned-clocks = <&gcc GCC_PCIE_1_AUX_CLK>; assigned-clock-rates = <19200000>; - iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, - <0x100 &apps_smmu 0x1c81 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c80 0x0 0x1>, + <0x100 &apps_smmu 0x1c81 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>; reset-names = "pci"; @@ -2126,8 +2126,8 @@ assigned-clocks = <&gcc GCC_PCIE_2_AUX_CLK>; assigned-clock-rates = <19200000>; - iommu-map = <0x0 &apps_smmu 0x1d00 0x1>, - <0x100 &apps_smmu 0x1d01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1d00 0x0 0x1>, + <0x100 &apps_smmu 0x1d01 0x0 0x1>; resets = <&gcc GCC_PCIE_2_BCR>; reset-names = "pci"; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts b/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts index dcdeefd2872837..ee85d5e09520e9 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts +++ b/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts @@ -182,7 +182,7 @@ regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; - gpio = <&pmc8280_1_gpios 2 GPIO_ACTIVE_HIGH>; + gpio = <&pmc8280_1_gpios 1 GPIO_ACTIVE_HIGH>; enable-active-high; pinctrl-names = "default"; @@ -915,7 +915,7 @@ }; misc_3p3_reg_en: misc-3p3-reg-en-state { - pins = "gpio2"; + pins = "gpio1"; function = "normal"; }; }; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts index abd9c5a67b9ffe..3ddd44e16e67fb 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts +++ b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts @@ -1555,7 +1555,7 @@ mclk-pins { pins = "gpio17"; function = "cam_mclk"; - drive-strength = <16>; + drive-strength = <6>; bias-disable; }; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts index f2b4470d4407fb..aa79704b5d5524 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts +++ b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts @@ -243,7 +243,7 @@ usb0-sbu-mux { compatible = "pericom,pi3usb102", "gpio-sbu-mux"; - enable-gpios = <&tlmm 101 GPIO_ACTIVE_LOW>; + enable-gpios = <&tlmm 100 GPIO_ACTIVE_LOW>; select-gpios = <&tlmm 164 GPIO_ACTIVE_HIGH>; pinctrl-0 = <&usb0_sbu_default>; @@ -996,7 +996,7 @@ usb0_sbu_default: usb0-sbu-state { oe-n-pins { - pins = "gpio101"; + pins = "gpio100"; function = "gpio"; bias-disable; drive-strength = <16>; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts index 125af356e24b96..0cea30e76d5fb9 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts +++ b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts @@ -763,20 +763,11 @@ }; &pmk8280_rtc { - nvmem-cells = <&rtc_offset>; - nvmem-cell-names = "offset"; + qcom,uefi-rtc-info; status = "okay"; }; -&pmk8280_sdam_6 { - status = "okay"; - - rtc_offset: rtc-offset@bc { - reg = <0xbc 0x4>; - }; -}; - &pmk8280_vadc { channel@144 { reg = ; @@ -1003,8 +994,8 @@ }; &usb_0_qmpphy { - vdda-phy-supply = <&vreg_l4d>; - vdda-pll-supply = <&vreg_l9d>; + vdda-phy-supply = <&vreg_l9d>; + vdda-pll-supply = <&vreg_l4d>; orientation-switch; @@ -1040,8 +1031,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l3b>; - vdda-pll-supply = <&vreg_l4b>; + vdda-phy-supply = <&vreg_l4b>; + vdda-pll-supply = <&vreg_l3b>; orientation-switch; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi index b0de9e262f299c..d7d1279008fa65 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi +++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi @@ -692,11 +692,6 @@ no-map; }; - pil_gpu_mem: gpu-mem@8bf00000 { - reg = <0 0x8bf00000 0 0x2000>; - no-map; - }; - pil_adsp_mem: adsp-region@86c00000 { reg = <0 0x86c00000 0 0x2000000>; no-map; @@ -712,6 +707,11 @@ no-map; }; + pil_gpu_mem: gpu-mem@8bf00000 { + reg = <0 0x8bf00000 0 0x2000>; + no-map; + }; + pil_nsp1_mem: cdsp1-region@8c600000 { reg = <0 0x8c600000 0 0x1e00000>; no-map; diff --git a/arch/arm64/boot/dts/qcom/sdm632-motorola-ocean.dts b/arch/arm64/boot/dts/qcom/sdm632-motorola-ocean.dts index 2f55db0c8ce35b..9ea3e5e76bf9e2 100644 --- a/arch/arm64/boot/dts/qcom/sdm632-motorola-ocean.dts +++ b/arch/arm64/boot/dts/qcom/sdm632-motorola-ocean.dts @@ -130,7 +130,7 @@ led: led@1 { reg = <1>; - default-trigger = "backlight"; + linux,default-trigger = "backlight"; function = LED_FUNCTION_BACKLIGHT; led-sources = <0 1 2>; }; diff --git a/arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi b/arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi index 0f57b915186b7f..b4854801a5f5e9 100644 --- a/arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi @@ -522,6 +522,11 @@ }; }; +&lpi_tlmm { + /* sensor gpios are protected */ + gpio-reserved-ranges = <0 8>, <12 6>; +}; + &mdss { status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts index 02416812b6a7f9..846138149b2cd3 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts @@ -1121,8 +1121,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l26a_1p2>; - vdda-pll-supply = <&vreg_l1a_0p875>; + vdda-phy-supply = <&vreg_l1a_0p875>; + vdda-pll-supply = <&vreg_l26a_1p2>; }; &usb_2 { @@ -1147,8 +1147,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l26a_1p2>; - vdda-pll-supply = <&vreg_l1a_0p875>; + vdda-phy-supply = <&vreg_l1a_0p875>; + vdda-pll-supply = <&vreg_l26a_1p2>; }; &ufs_mem_hc { diff --git a/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi index b2dc0327dff856..ebeebbde640537 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi @@ -571,8 +571,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l26a_1p2>; - vdda-pll-supply = <&vreg_l1a_0p875>; + vdda-phy-supply = <&vreg_l1a_0p875>; + vdda-pll-supply = <&vreg_l26a_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi index e0c3566761bf8c..c14b4b3d9a3c94 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi @@ -677,8 +677,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vdda_usb1_ss_1p2>; - vdda-pll-supply = <&vdda_usb1_ss_core>; + vdda-phy-supply = <&vdda_usb1_ss_core>; + vdda-pll-supply = <&vdda_usb1_ss_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sdm845-mtp.dts b/arch/arm64/boot/dts/qcom/sdm845-mtp.dts index 091568642faa78..00ea30afc9f6d0 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-mtp.dts @@ -729,8 +729,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vdda_usb1_ss_1p2>; - vdda-pll-supply = <&vdda_usb1_ss_core>; + vdda-phy-supply = <&vdda_usb1_ss_core>; + vdda-pll-supply = <&vdda_usb1_ss_1p2>; }; &usb_2 { @@ -761,8 +761,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vdda_usb2_ss_1p2>; - vdda-pll-supply = <&vdda_usb2_ss_core>; + vdda-phy-supply = <&vdda_usb2_ss_core>; + vdda-pll-supply = <&vdda_usb2_ss_1p2>; }; &venus { diff --git a/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts b/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts index 0d915ecb0444f3..cd735417adddcd 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts @@ -949,8 +949,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vdda_usb1_ss_1p2>; - vdda-pll-supply = <&vdda_usb1_ss_core>; + vdda-phy-supply = <&vdda_usb1_ss_core>; + vdda-pll-supply = <&vdda_usb1_ss_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts b/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts index f21b914f22b517..ca503af9b832b7 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts @@ -876,8 +876,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l26a_1p2>; - vdda-pll-supply = <&vreg_l1a_0p875>; + vdda-phy-supply = <&vreg_l1a_0p875>; + vdda-pll-supply = <&vreg_l26a_1p2>; }; &venus { diff --git a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi index 54e58ca0469344..710b296e496431 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi @@ -645,8 +645,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l26a_1p2>; - vdda-pll-supply = <&vreg_l1a_0p875>; + vdda-phy-supply = <&vreg_l1a_0p875>; + vdda-pll-supply = <&vreg_l26a_1p2>; }; &venus { diff --git a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts index a44d6e776c82ef..cce7348e583ac0 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts @@ -668,8 +668,8 @@ }; &usb_1_qmpphy { - vdda-pll-supply = <&vreg_l1a_0p875>; - vdda-phy-supply = <&vreg_l26a_1p2>; + vdda-pll-supply = <&vreg_l26a_1p2>; + vdda-phy-supply = <&vreg_l1a_0p875>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sdm845.dtsi b/arch/arm64/boot/dts/qcom/sdm845.dtsi index 4ae8627d6dbc3d..7c2a439dffdb1c 100644 --- a/arch/arm64/boot/dts/qcom/sdm845.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845.dtsi @@ -2372,22 +2372,22 @@ "slave_q2a", "tbu"; - iommu-map = <0x0 &apps_smmu 0x1c10 0x1>, - <0x100 &apps_smmu 0x1c11 0x1>, - <0x200 &apps_smmu 0x1c12 0x1>, - <0x300 &apps_smmu 0x1c13 0x1>, - <0x400 &apps_smmu 0x1c14 0x1>, - <0x500 &apps_smmu 0x1c15 0x1>, - <0x600 &apps_smmu 0x1c16 0x1>, - <0x700 &apps_smmu 0x1c17 0x1>, - <0x800 &apps_smmu 0x1c18 0x1>, - <0x900 &apps_smmu 0x1c19 0x1>, - <0xa00 &apps_smmu 0x1c1a 0x1>, - <0xb00 &apps_smmu 0x1c1b 0x1>, - <0xc00 &apps_smmu 0x1c1c 0x1>, - <0xd00 &apps_smmu 0x1c1d 0x1>, - <0xe00 &apps_smmu 0x1c1e 0x1>, - <0xf00 &apps_smmu 0x1c1f 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c10 0x0 0x1>, + <0x100 &apps_smmu 0x1c11 0x0 0x1>, + <0x200 &apps_smmu 0x1c12 0x0 0x1>, + <0x300 &apps_smmu 0x1c13 0x0 0x1>, + <0x400 &apps_smmu 0x1c14 0x0 0x1>, + <0x500 &apps_smmu 0x1c15 0x0 0x1>, + <0x600 &apps_smmu 0x1c16 0x0 0x1>, + <0x700 &apps_smmu 0x1c17 0x0 0x1>, + <0x800 &apps_smmu 0x1c18 0x0 0x1>, + <0x900 &apps_smmu 0x1c19 0x0 0x1>, + <0xa00 &apps_smmu 0x1c1a 0x0 0x1>, + <0xb00 &apps_smmu 0x1c1b 0x0 0x1>, + <0xc00 &apps_smmu 0x1c1c 0x0 0x1>, + <0xd00 &apps_smmu 0x1c1d 0x0 0x1>, + <0xe00 &apps_smmu 0x1c1e 0x0 0x1>, + <0xf00 &apps_smmu 0x1c1f 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -2502,22 +2502,22 @@ assigned-clocks = <&gcc GCC_PCIE_1_AUX_CLK>; assigned-clock-rates = <19200000>; - iommu-map = <0x0 &apps_smmu 0x1c00 0x1>, - <0x100 &apps_smmu 0x1c01 0x1>, - <0x200 &apps_smmu 0x1c02 0x1>, - <0x300 &apps_smmu 0x1c03 0x1>, - <0x400 &apps_smmu 0x1c04 0x1>, - <0x500 &apps_smmu 0x1c05 0x1>, - <0x600 &apps_smmu 0x1c06 0x1>, - <0x700 &apps_smmu 0x1c07 0x1>, - <0x800 &apps_smmu 0x1c08 0x1>, - <0x900 &apps_smmu 0x1c09 0x1>, - <0xa00 &apps_smmu 0x1c0a 0x1>, - <0xb00 &apps_smmu 0x1c0b 0x1>, - <0xc00 &apps_smmu 0x1c0c 0x1>, - <0xd00 &apps_smmu 0x1c0d 0x1>, - <0xe00 &apps_smmu 0x1c0e 0x1>, - <0xf00 &apps_smmu 0x1c0f 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c00 0x0 0x1>, + <0x100 &apps_smmu 0x1c01 0x0 0x1>, + <0x200 &apps_smmu 0x1c02 0x0 0x1>, + <0x300 &apps_smmu 0x1c03 0x0 0x1>, + <0x400 &apps_smmu 0x1c04 0x0 0x1>, + <0x500 &apps_smmu 0x1c05 0x0 0x1>, + <0x600 &apps_smmu 0x1c06 0x0 0x1>, + <0x700 &apps_smmu 0x1c07 0x0 0x1>, + <0x800 &apps_smmu 0x1c08 0x0 0x1>, + <0x900 &apps_smmu 0x1c09 0x0 0x1>, + <0xa00 &apps_smmu 0x1c0a 0x0 0x1>, + <0xb00 &apps_smmu 0x1c0b 0x0 0x1>, + <0xc00 &apps_smmu 0x1c0c 0x0 0x1>, + <0xd00 &apps_smmu 0x1c0d 0x0 0x1>, + <0xe00 &apps_smmu 0x1c0e 0x0 0x1>, + <0xf00 &apps_smmu 0x1c0f 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>; reset-names = "pci"; diff --git a/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts b/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts index f048653818702a..a4339b562b6046 100644 --- a/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts +++ b/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts @@ -872,8 +872,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vdda_usb1_ss_1p2>; - vdda-pll-supply = <&vdda_usb1_ss_core>; + vdda-phy-supply = <&vdda_usb1_ss_core>; + vdda-pll-supply = <&vdda_usb1_ss_1p2>; status = "okay"; }; @@ -898,8 +898,8 @@ }; &usb_2_qmpphy { - vdda-phy-supply = <&vdda_usb2_ss_1p2>; - vdda-pll-supply = <&vdda_usb2_ss_core>; + vdda-phy-supply = <&vdda_usb2_ss_core>; + vdda-pll-supply = <&vdda_usb2_ss_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts index 8cdaef5360d112..3c353d27d7e892 100644 --- a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts +++ b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts @@ -864,8 +864,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vdda_usb1_ss_1p2>; - vdda-pll-supply = <&vdda_usb1_ss_core>; + vdda-phy-supply = <&vdda_usb1_ss_core>; + vdda-pll-supply = <&vdda_usb1_ss_1p2>; }; &usb_1_qmpphy_out { @@ -957,8 +957,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vdda_usb2_ss_1p2>; - vdda-pll-supply = <&vdda_usb2_ss_core>; + vdda-phy-supply = <&vdda_usb2_ss_core>; + vdda-pll-supply = <&vdda_usb2_ss_1p2>; }; &venus { diff --git a/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts b/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts index a676d3ea01b997..b7a1e2317fc2f3 100644 --- a/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts +++ b/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts @@ -609,8 +609,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vdda_usb1_ss_1p2>; - vdda-pll-supply = <&vdda_usb1_ss_core>; + vdda-phy-supply = <&vdda_usb1_ss_core>; + vdda-pll-supply = <&vdda_usb1_ss_1p2>; }; &usb_2 { @@ -635,8 +635,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vdda_usb2_ss_1p2>; - vdda-pll-supply = <&vdda_usb2_ss_core>; + vdda-phy-supply = <&vdda_usb2_ss_core>; + vdda-pll-supply = <&vdda_usb2_ss_1p2>; }; &venus { diff --git a/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts index b9d9439e230b45..139f2b401af509 100644 --- a/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts +++ b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts @@ -198,7 +198,7 @@ status = "okay"; panel@0 { - compatible = "samsung,s6e8fco-m1906f9"; + compatible = "samsung,s6e8fc0-m1906f9"; reg = <0>; reset-gpios = <&tlmm 90 GPIO_ACTIVE_LOW>; diff --git a/arch/arm64/boot/dts/qcom/sm6125.dtsi b/arch/arm64/boot/dts/qcom/sm6125.dtsi index 6e84c226948c04..a3caf5c87ae76e 100644 --- a/arch/arm64/boot/dts/qcom/sm6125.dtsi +++ b/arch/arm64/boot/dts/qcom/sm6125.dtsi @@ -374,22 +374,23 @@ }; soc@0 { - #address-cells = <1>; - #size-cells = <1>; - ranges = <0x00 0x00 0x00 0xffffffff>; + #address-cells = <2>; + #size-cells = <2>; + ranges = <0 0 0 0 0x10 0>; + dma-ranges = <0 0 0 0 0x10 0>; compatible = "simple-bus"; tcsr_mutex: hwlock@340000 { compatible = "qcom,tcsr-mutex"; - reg = <0x00340000 0x20000>; + reg = <0x0 0x00340000 0x0 0x20000>; #hwlock-cells = <1>; }; tlmm: pinctrl@500000 { compatible = "qcom,sm6125-tlmm"; - reg = <0x00500000 0x400000>, - <0x00900000 0x400000>, - <0x00d00000 0x400000>; + reg = <0x0 0x00500000 0x0 0x400000>, + <0x0 0x00900000 0x0 0x400000>, + <0x0 0x00d00000 0x0 0x400000>; reg-names = "west", "south", "east"; interrupts = ; gpio-controller; @@ -672,7 +673,7 @@ gcc: clock-controller@1400000 { compatible = "qcom,gcc-sm6125"; - reg = <0x01400000 0x1f0000>; + reg = <0x0 0x01400000 0x0 0x1f0000>; #clock-cells = <1>; #reset-cells = <1>; #power-domain-cells = <1>; @@ -682,7 +683,7 @@ hsusb_phy1: phy@1613000 { compatible = "qcom,msm8996-qusb2-phy"; - reg = <0x01613000 0x180>; + reg = <0x0 0x01613000 0x0 0x180>; #phy-cells = <0>; clocks = <&gcc GCC_AHB2PHY_USB_CLK>, @@ -695,18 +696,18 @@ rng: rng@1b53000 { compatible = "qcom,prng-ee"; - reg = <0x01b53000 0x1000>; + reg = <0x0 0x01b53000 0x0 0x1000>; clocks = <&gcc GCC_PRNG_AHB_CLK>; clock-names = "core"; }; spmi_bus: spmi@1c40000 { compatible = "qcom,spmi-pmic-arb"; - reg = <0x01c40000 0x1100>, - <0x01e00000 0x2000000>, - <0x03e00000 0x100000>, - <0x03f00000 0xa0000>, - <0x01c0a000 0x26000>; + reg = <0x0 0x01c40000 0x0 0x1100>, + <0x0 0x01e00000 0x0 0x2000000>, + <0x0 0x03e00000 0x0 0x100000>, + <0x0 0x03f00000 0x0 0xa0000>, + <0x0 0x01c0a000 0x0 0x26000>; reg-names = "core", "chnls", "obsrvr", "intr", "cnfg"; interrupt-names = "periph_irq"; interrupts = ; @@ -720,12 +721,13 @@ rpm_msg_ram: sram@45f0000 { compatible = "qcom,rpm-msg-ram"; - reg = <0x045f0000 0x7000>; + reg = <0x0 0x045f0000 0x0 0x7000>; }; sdhc_1: mmc@4744000 { compatible = "qcom,sm6125-sdhci", "qcom,sdhci-msm-v5"; - reg = <0x04744000 0x1000>, <0x04745000 0x1000>; + reg = <0x0 0x04744000 0x0 0x1000>, + <0x0 0x04745000 0x0 0x1000>; reg-names = "hc", "cqhci"; interrupts = , @@ -752,7 +754,7 @@ sdhc_2: mmc@4784000 { compatible = "qcom,sm6125-sdhci", "qcom,sdhci-msm-v5"; - reg = <0x04784000 0x1000>; + reg = <0x0 0x04784000 0x0 0x1000>; reg-names = "hc"; interrupts = , @@ -780,7 +782,8 @@ ufs_mem_hc: ufshc@4804000 { compatible = "qcom,sm6125-ufshc", "qcom,ufshc", "jedec,ufs-2.0"; - reg = <0x04804000 0x3000>, <0x04810000 0x8000>; + reg = <0x0 0x04804000 0x0 0x3000>, + <0x0 0x04810000 0x0 0x8000>; reg-names = "std", "ice"; interrupts = ; @@ -825,7 +828,7 @@ ufs_mem_phy: phy@4807000 { compatible = "qcom,sm6125-qmp-ufs-phy"; - reg = <0x04807000 0xdb8>; + reg = <0x0 0x04807000 0x0 0xdb8>; clocks = <&rpmcc RPM_SMD_XO_CLK_SRC>, <&gcc GCC_UFS_PHY_PHY_AUX_CLK>, @@ -846,7 +849,7 @@ gpi_dma0: dma-controller@4a00000 { compatible = "qcom,sm6125-gpi-dma", "qcom,sdm845-gpi-dma"; - reg = <0x04a00000 0x60000>; + reg = <0x0 0x04a00000 0x0 0x60000>; interrupts = , , , @@ -864,19 +867,19 @@ qupv3_id_0: geniqup@4ac0000 { compatible = "qcom,geni-se-qup"; - reg = <0x04ac0000 0x2000>; + reg = <0x0 0x04ac0000 0x0 0x2000>; clocks = <&gcc GCC_QUPV3_WRAP_0_M_AHB_CLK>, <&gcc GCC_QUPV3_WRAP_0_S_AHB_CLK>; clock-names = "m-ahb", "s-ahb"; iommus = <&apps_smmu 0x123 0x0>; - #address-cells = <1>; - #size-cells = <1>; + #address-cells = <2>; + #size-cells = <2>; ranges; status = "disabled"; i2c0: i2c@4a80000 { compatible = "qcom,geni-i2c"; - reg = <0x04a80000 0x4000>; + reg = <0x0 0x04a80000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP0_S0_CLK>; clock-names = "se"; interrupts = ; @@ -893,7 +896,7 @@ spi0: spi@4a80000 { compatible = "qcom,geni-spi"; - reg = <0x04a80000 0x4000>; + reg = <0x0 0x04a80000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP0_S0_CLK>; clock-names = "se"; interrupts = ; @@ -910,7 +913,7 @@ i2c1: i2c@4a84000 { compatible = "qcom,geni-i2c"; - reg = <0x04a84000 0x4000>; + reg = <0x0 0x04a84000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP0_S1_CLK>; clock-names = "se"; interrupts = ; @@ -927,7 +930,7 @@ i2c2: i2c@4a88000 { compatible = "qcom,geni-i2c"; - reg = <0x04a88000 0x4000>; + reg = <0x0 0x04a88000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>; clock-names = "se"; interrupts = ; @@ -944,7 +947,7 @@ spi2: spi@4a88000 { compatible = "qcom,geni-spi"; - reg = <0x04a88000 0x4000>; + reg = <0x0 0x04a88000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>; clock-names = "se"; interrupts = ; @@ -961,7 +964,7 @@ i2c3: i2c@4a8c000 { compatible = "qcom,geni-i2c"; - reg = <0x04a8c000 0x4000>; + reg = <0x0 0x04a8c000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP0_S3_CLK>; clock-names = "se"; interrupts = ; @@ -978,7 +981,7 @@ i2c4: i2c@4a90000 { compatible = "qcom,geni-i2c"; - reg = <0x04a90000 0x4000>; + reg = <0x0 0x04a90000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP0_S4_CLK>; clock-names = "se"; interrupts = ; @@ -995,7 +998,7 @@ uart4: serial@4a90000 { compatible = "qcom,geni-debug-uart"; - reg = <0x04a90000 0x4000>; + reg = <0x0 0x04a90000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP0_S4_CLK>; clock-names = "se"; interrupts = ; @@ -1007,7 +1010,7 @@ gpi_dma1: dma-controller@4c00000 { compatible = "qcom,sm6125-gpi-dma", "qcom,sdm845-gpi-dma"; - reg = <0x04c00000 0x60000>; + reg = <0x0 0x04c00000 0x0 0x60000>; interrupts = , , , @@ -1025,19 +1028,19 @@ qupv3_id_1: geniqup@4cc0000 { compatible = "qcom,geni-se-qup"; - reg = <0x04cc0000 0x2000>; + reg = <0x0 0x04cc0000 0x0 0x2000>; clocks = <&gcc GCC_QUPV3_WRAP_1_M_AHB_CLK>, <&gcc GCC_QUPV3_WRAP_1_S_AHB_CLK>; clock-names = "m-ahb", "s-ahb"; iommus = <&apps_smmu 0x143 0x0>; - #address-cells = <1>; - #size-cells = <1>; + #address-cells = <2>; + #size-cells = <2>; ranges; status = "disabled"; i2c5: i2c@4c80000 { compatible = "qcom,geni-i2c"; - reg = <0x04c80000 0x4000>; + reg = <0x0 0x04c80000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP1_S0_CLK>; clock-names = "se"; interrupts = ; @@ -1054,7 +1057,7 @@ spi5: spi@4c80000 { compatible = "qcom,geni-spi"; - reg = <0x04c80000 0x4000>; + reg = <0x0 0x04c80000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP1_S0_CLK>; clock-names = "se"; interrupts = ; @@ -1071,7 +1074,7 @@ i2c6: i2c@4c84000 { compatible = "qcom,geni-i2c"; - reg = <0x04c84000 0x4000>; + reg = <0x0 0x04c84000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP1_S1_CLK>; clock-names = "se"; interrupts = ; @@ -1088,7 +1091,7 @@ spi6: spi@4c84000 { compatible = "qcom,geni-spi"; - reg = <0x04c84000 0x4000>; + reg = <0x0 0x04c84000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP1_S1_CLK>; clock-names = "se"; interrupts = ; @@ -1105,7 +1108,7 @@ i2c7: i2c@4c88000 { compatible = "qcom,geni-i2c"; - reg = <0x04c88000 0x4000>; + reg = <0x0 0x04c88000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP1_S2_CLK>; clock-names = "se"; interrupts = ; @@ -1122,7 +1125,7 @@ i2c8: i2c@4c8c000 { compatible = "qcom,geni-i2c"; - reg = <0x04c8c000 0x4000>; + reg = <0x0 0x04c8c000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP1_S3_CLK>; clock-names = "se"; interrupts = ; @@ -1139,7 +1142,7 @@ spi8: spi@4c8c000 { compatible = "qcom,geni-spi"; - reg = <0x04c8c000 0x4000>; + reg = <0x0 0x04c8c000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP1_S3_CLK>; clock-names = "se"; interrupts = ; @@ -1156,7 +1159,7 @@ i2c9: i2c@4c90000 { compatible = "qcom,geni-i2c"; - reg = <0x04c90000 0x4000>; + reg = <0x0 0x04c90000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP1_S4_CLK>; clock-names = "se"; interrupts = ; @@ -1173,7 +1176,7 @@ spi9: spi@4c90000 { compatible = "qcom,geni-spi"; - reg = <0x04c90000 0x4000>; + reg = <0x0 0x04c90000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP1_S4_CLK>; clock-names = "se"; interrupts = ; @@ -1191,9 +1194,9 @@ usb3: usb@4ef8800 { compatible = "qcom,sm6125-dwc3", "qcom,dwc3"; - reg = <0x04ef8800 0x400>; - #address-cells = <1>; - #size-cells = <1>; + reg = <0x0 0x04ef8800 0x0 0x400>; + #address-cells = <2>; + #size-cells = <2>; ranges; clocks = <&gcc GCC_CFG_NOC_USB3_PRIM_AXI_CLK>, @@ -1228,7 +1231,7 @@ usb3_dwc3: usb@4e00000 { compatible = "snps,dwc3"; - reg = <0x04e00000 0xcd00>; + reg = <0x0 0x04e00000 0x0 0xcd00>; interrupts = ; iommus = <&apps_smmu 0x100 0x0>; phys = <&hsusb_phy1>; @@ -1244,12 +1247,12 @@ sram@4690000 { compatible = "qcom,rpm-stats"; - reg = <0x04690000 0x10000>; + reg = <0x0 0x04690000 0x0 0x10000>; }; mdss: display-subsystem@5e00000 { compatible = "qcom,sm6125-mdss"; - reg = <0x05e00000 0x1000>; + reg = <0x0 0x05e00000 0x0 0x1000>; reg-names = "mdss"; interrupts = ; @@ -1269,16 +1272,16 @@ iommus = <&apps_smmu 0x400 0x0>; - #address-cells = <1>; - #size-cells = <1>; + #address-cells = <2>; + #size-cells = <2>; ranges; status = "disabled"; mdss_mdp: display-controller@5e01000 { compatible = "qcom,sm6125-dpu"; - reg = <0x05e01000 0x83208>, - <0x05eb0000 0x3000>; + reg = <0x0 0x05e01000 0x0 0x83208>, + <0x0 0x05eb0000 0x0 0x3000>; reg-names = "mdp", "vbif"; interrupt-parent = <&mdss>; @@ -1348,7 +1351,7 @@ mdss_dsi0: dsi@5e94000 { compatible = "qcom,sm6125-dsi-ctrl", "qcom,mdss-dsi-ctrl"; - reg = <0x05e94000 0x400>; + reg = <0x0 0x05e94000 0x0 0x400>; reg-names = "dsi_ctrl"; interrupt-parent = <&mdss>; @@ -1417,9 +1420,9 @@ mdss_dsi0_phy: phy@5e94400 { compatible = "qcom,sm6125-dsi-phy-14nm"; - reg = <0x05e94400 0x100>, - <0x05e94500 0x300>, - <0x05e94800 0x188>; + reg = <0x0 0x05e94400 0x0 0x100>, + <0x0 0x05e94500 0x0 0x300>, + <0x0 0x05e94800 0x0 0x188>; reg-names = "dsi_phy", "dsi_phy_lane", "dsi_pll"; @@ -1441,7 +1444,7 @@ dispcc: clock-controller@5f00000 { compatible = "qcom,sm6125-dispcc"; - reg = <0x05f00000 0x20000>; + reg = <0x0 0x05f00000 0x0 0x20000>; clocks = <&rpmcc RPM_SMD_XO_CLK_SRC>, <&mdss_dsi0_phy DSI_BYTE_PLL_CLK>, @@ -1470,7 +1473,7 @@ apps_smmu: iommu@c600000 { compatible = "qcom,sm6125-smmu-500", "qcom,smmu-500", "arm,mmu-500"; - reg = <0x0c600000 0x80000>; + reg = <0x0 0x0c600000 0x0 0x80000>; interrupts = , , , @@ -1544,74 +1547,74 @@ apcs_glb: mailbox@f111000 { compatible = "qcom,sm6125-apcs-hmss-global", "qcom,msm8994-apcs-kpss-global"; - reg = <0x0f111000 0x1000>; + reg = <0x0 0x0f111000 0x0 0x1000>; #mbox-cells = <1>; }; timer@f120000 { compatible = "arm,armv7-timer-mem"; - #address-cells = <1>; + #address-cells = <2>; #size-cells = <1>; - ranges; - reg = <0x0f120000 0x1000>; + reg = <0x0 0x0f120000 0x0 0x1000>; + ranges = <0x0 0x0 0x0 0x0 0x20000000>; clock-frequency = <19200000>; frame@f121000 { frame-number = <0>; interrupts = , ; - reg = <0x0f121000 0x1000>, - <0x0f122000 0x1000>; + reg = <0x0 0x0f121000 0x1000>, + <0x0 0x0f122000 0x1000>; }; frame@f123000 { frame-number = <1>; interrupts = ; - reg = <0x0f123000 0x1000>; + reg = <0x0 0x0f123000 0x1000>; status = "disabled"; }; frame@f124000 { frame-number = <2>; interrupts = ; - reg = <0x0f124000 0x1000>; + reg = <0x0 0x0f124000 0x1000>; status = "disabled"; }; frame@f125000 { frame-number = <3>; interrupts = ; - reg = <0x0f125000 0x1000>; + reg = <0x0 0x0f125000 0x1000>; status = "disabled"; }; frame@f126000 { frame-number = <4>; interrupts = ; - reg = <0x0f126000 0x1000>; + reg = <0x0 0x0f126000 0x1000>; status = "disabled"; }; frame@f127000 { frame-number = <5>; interrupts = ; - reg = <0x0f127000 0x1000>; + reg = <0x0 0x0f127000 0x1000>; status = "disabled"; }; frame@f128000 { frame-number = <6>; interrupts = ; - reg = <0x0f128000 0x1000>; + reg = <0x0 0x0f128000 0x1000>; status = "disabled"; }; }; intc: interrupt-controller@f200000 { compatible = "arm,gic-v3"; - reg = <0x0f200000 0x20000>, - <0x0f300000 0x100000>; + reg = <0x0 0x0f200000 0x0 0x20000>, + <0x0 0x0f300000 0x0 0x100000>; #interrupt-cells = <3>; interrupt-controller; interrupts = ; diff --git a/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts b/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts index 23f950067a08b5..ce97e5bddecb13 100644 --- a/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts +++ b/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts @@ -48,7 +48,7 @@ stdout-path = "serial0:115200n8"; - framebuffer0: framebuffer@a000000 { + framebuffer0: framebuffer@a0000000 { compatible = "simple-framebuffer"; reg = <0 0xa0000000 0 (2340 * 1080 * 4)>; width = <1080>; @@ -1247,8 +1247,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l22a>; - vdda-pll-supply = <&vreg_l16a>; + vdda-phy-supply = <&vreg_l16a>; + vdda-pll-supply = <&vreg_l22a>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8150-hdk.dts b/arch/arm64/boot/dts/qcom/sm8150-hdk.dts index 8fb04ce72f91e3..cdced5c848d3a3 100644 --- a/arch/arm64/boot/dts/qcom/sm8150-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8150-hdk.dts @@ -801,8 +801,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l18a_0p8>; + vdda-phy-supply = <&vreg_l18a_0p8>; + vdda-pll-supply = <&vreg_l3c_1p2>; orientation-switch; }; @@ -812,8 +812,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l5a_0p875>; + vdda-phy-supply = <&vreg_l5a_0p875>; + vdda-pll-supply = <&vreg_l3c_1p2>; }; &usb_1 { diff --git a/arch/arm64/boot/dts/qcom/sm8150-microsoft-surface-duo.dts b/arch/arm64/boot/dts/qcom/sm8150-microsoft-surface-duo.dts index 835ef929ff2df4..f37e11bde067d7 100644 --- a/arch/arm64/boot/dts/qcom/sm8150-microsoft-surface-duo.dts +++ b/arch/arm64/boot/dts/qcom/sm8150-microsoft-surface-duo.dts @@ -520,8 +520,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vdda_usb_ss_dp_core_1>; + vdda-phy-supply = <&vdda_usb_ss_dp_core_1>; + vdda-pll-supply = <&vreg_l3c_1p2>; }; &usb_1 { diff --git a/arch/arm64/boot/dts/qcom/sm8150-mtp.dts b/arch/arm64/boot/dts/qcom/sm8150-mtp.dts index 0f2d511624a8bf..6a6564d8d7f7ca 100644 --- a/arch/arm64/boot/dts/qcom/sm8150-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sm8150-mtp.dts @@ -438,8 +438,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vdda_usb_ss_dp_core_1>; + vdda-phy-supply = <&vdda_usb_ss_dp_core_1>; + vdda-pll-supply = <&vreg_l3c_1p2>; }; &usb_1 { diff --git a/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano.dtsi b/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano.dtsi index 70fd6455518b91..6c2e2bf869d0d1 100644 --- a/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano.dtsi @@ -869,6 +869,6 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l18a_0p8>; + vdda-phy-supply = <&vreg_l18a_0p8>; + vdda-pll-supply = <&vreg_l3c_1p2>; }; diff --git a/arch/arm64/boot/dts/qcom/sm8150.dtsi b/arch/arm64/boot/dts/qcom/sm8150.dtsi index 0e101096209abb..109a76f4ca75f9 100644 --- a/arch/arm64/boot/dts/qcom/sm8150.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8150.dtsi @@ -1893,8 +1893,8 @@ "bus_slave", "slave_q2a"; - iommu-map = <0x0 &apps_smmu 0x1d80 0x1>, - <0x100 &apps_smmu 0x1d81 0x1>; + iommu-map = <0x0 &apps_smmu 0x1d80 0x0 0x1>, + <0x100 &apps_smmu 0x1d81 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -2011,8 +2011,8 @@ assigned-clocks = <&gcc GCC_PCIE_1_AUX_CLK>; assigned-clock-rates = <19200000>; - iommu-map = <0x0 &apps_smmu 0x1e00 0x1>, - <0x100 &apps_smmu 0x1e01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1e00 0x0 0x1>, + <0x100 &apps_smmu 0x1e01 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>; reset-names = "pci"; diff --git a/arch/arm64/boot/dts/qcom/sm8250-hdk.dts b/arch/arm64/boot/dts/qcom/sm8250-hdk.dts index 3ea9d2b1a7d581..c1c673086df510 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8250-hdk.dts @@ -438,14 +438,14 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l9a_1p2>; - vdda-pll-supply = <&vreg_l18a_0p92>; + vdda-phy-supply = <&vreg_l18a_0p92>; + vdda-pll-supply = <&vreg_l9a_1p2>; }; &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l9a_1p2>; - vdda-pll-supply = <&vreg_l18a_0p92>; + vdda-phy-supply = <&vreg_l18a_0p92>; + vdda-pll-supply = <&vreg_l9a_1p2>; }; &usb_1 { diff --git a/arch/arm64/boot/dts/qcom/sm8250-mtp.dts b/arch/arm64/boot/dts/qcom/sm8250-mtp.dts index 51779b99176d06..14c8ca602110f2 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sm8250-mtp.dts @@ -862,8 +862,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l9a_1p2>; - vdda-pll-supply = <&vreg_l18a_0p9>; + vdda-phy-supply = <&vreg_l18a_0p9>; + vdda-pll-supply = <&vreg_l9a_1p2>; }; &usb_2 { @@ -885,8 +885,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l9a_1p2>; - vdda-pll-supply = <&vreg_l18a_0p9>; + vdda-phy-supply = <&vreg_l18a_0p9>; + vdda-pll-supply = <&vreg_l9a_1p2>; }; &venus { diff --git a/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo.dtsi b/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo.dtsi index d8289b2698f37b..51c0caaf4ea126 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo.dtsi @@ -715,8 +715,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l9a_1p2>; - vdda-pll-supply = <&vreg_l18a_0p9>; + vdda-phy-supply = <&vreg_l18a_0p9>; + vdda-pll-supply = <&vreg_l9a_1p2>; }; &venus { diff --git a/arch/arm64/boot/dts/qcom/sm8250-xiaomi-elish-common.dtsi b/arch/arm64/boot/dts/qcom/sm8250-xiaomi-elish-common.dtsi index 51b57c697a7539..3d48467c52c99c 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-xiaomi-elish-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8250-xiaomi-elish-common.dtsi @@ -28,7 +28,7 @@ /* required for bootloader to select correct board */ qcom,msm-id = ; /* SM8250 v2.1 */ - qcom,board-id = <0x10008 0>; + qcom,board-id = <0x2f 0>; aliases { serial0 = &uart6; diff --git a/arch/arm64/boot/dts/qcom/sm8250-xiaomi-pipa.dts b/arch/arm64/boot/dts/qcom/sm8250-xiaomi-pipa.dts index 078ba13f8762e0..f11e4e244dffd3 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-xiaomi-pipa.dts +++ b/arch/arm64/boot/dts/qcom/sm8250-xiaomi-pipa.dts @@ -515,8 +515,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l9a_1p2>; - vdda-pll-supply = <&vreg_l18a_0p9>; + vdda-phy-supply = <&vreg_l18a_0p9>; + vdda-pll-supply = <&vreg_l9a_1p2>; }; &ufs_mem_hc { diff --git a/arch/arm64/boot/dts/qcom/sm8250.dtsi b/arch/arm64/boot/dts/qcom/sm8250.dtsi index 7076720413ab2c..ab461b5883c737 100644 --- a/arch/arm64/boot/dts/qcom/sm8250.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8250.dtsi @@ -2190,8 +2190,8 @@ "tbu", "ddrss_sf_tbu"; - iommu-map = <0x0 &apps_smmu 0x1c00 0x1>, - <0x100 &apps_smmu 0x1c01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c00 0x0 0x1>, + <0x100 &apps_smmu 0x1c01 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -2317,8 +2317,8 @@ assigned-clocks = <&gcc GCC_PCIE_1_AUX_CLK>; assigned-clock-rates = <19200000>; - iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, - <0x100 &apps_smmu 0x1c81 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c80 0x0 0x1>, + <0x100 &apps_smmu 0x1c81 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>; reset-names = "pci"; @@ -2444,8 +2444,8 @@ assigned-clocks = <&gcc GCC_PCIE_2_AUX_CLK>; assigned-clock-rates = <19200000>; - iommu-map = <0x0 &apps_smmu 0x1d00 0x1>, - <0x100 &apps_smmu 0x1d01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1d00 0x0 0x1>, + <0x100 &apps_smmu 0x1d01 0x0 0x1>; resets = <&gcc GCC_PCIE_2_BCR>; reset-names = "pci"; @@ -4326,8 +4326,12 @@ interrupts = ; power-domains = <&videocc MVS0C_GDSC>, <&videocc MVS0_GDSC>, - <&rpmhpd RPMHPD_MX>; - power-domain-names = "venus", "vcodec0", "mx"; + <&rpmhpd RPMHPD_MX>, + <&rpmhpd RPMHPD_MMCX>; + power-domain-names = "venus", + "vcodec0", + "mx", + "mmcx"; operating-points-v2 = <&venus_opp_table>; clocks = <&gcc GCC_VIDEO_AXI0_CLK>, @@ -4351,24 +4355,28 @@ venus_opp_table: opp-table { compatible = "operating-points-v2"; - opp-720000000 { - opp-hz = /bits/ 64 <720000000>; - required-opps = <&rpmhpd_opp_low_svs>; + opp-240000000 { + opp-hz = /bits/ 64 <240000000>; + required-opps = <&rpmhpd_opp_svs>, + <&rpmhpd_opp_low_svs>; }; - opp-1014000000 { - opp-hz = /bits/ 64 <1014000000>; - required-opps = <&rpmhpd_opp_svs>; + opp-338000000 { + opp-hz = /bits/ 64 <338000000>; + required-opps = <&rpmhpd_opp_svs>, + <&rpmhpd_opp_svs>; }; - opp-1098000000 { - opp-hz = /bits/ 64 <1098000000>; - required-opps = <&rpmhpd_opp_svs_l1>; + opp-366000000 { + opp-hz = /bits/ 64 <366000000>; + required-opps = <&rpmhpd_opp_svs_l1>, + <&rpmhpd_opp_svs_l1>; }; - opp-1332000000 { - opp-hz = /bits/ 64 <1332000000>; - required-opps = <&rpmhpd_opp_nom>; + opp-444000000 { + opp-hz = /bits/ 64 <444000000>; + required-opps = <&rpmhpd_opp_svs_l1>, + <&rpmhpd_opp_nom>; }; }; }; diff --git a/arch/arm64/boot/dts/qcom/sm8350-hdk.dts b/arch/arm64/boot/dts/qcom/sm8350-hdk.dts index 5f975d00946585..b43dbb5921b187 100644 --- a/arch/arm64/boot/dts/qcom/sm8350-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8350-hdk.dts @@ -860,8 +860,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p88>; + vdda-phy-supply = <&vreg_l1b_0p88>; + vdda-pll-supply = <&vreg_l6b_1p2>; }; &usb_1_qmpphy_out { @@ -890,8 +890,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l5b_0p88>; + vdda-phy-supply = <&vreg_l5b_0p88>; + vdda-pll-supply = <&vreg_l6b_1p2>; }; /* PINCTRL - additions to nodes defined in sm8350.dtsi */ diff --git a/arch/arm64/boot/dts/qcom/sm8350-microsoft-surface-duo2.dts b/arch/arm64/boot/dts/qcom/sm8350-microsoft-surface-duo2.dts index 3bd5e57cbcdaa3..57bb8c322e8b01 100644 --- a/arch/arm64/boot/dts/qcom/sm8350-microsoft-surface-duo2.dts +++ b/arch/arm64/boot/dts/qcom/sm8350-microsoft-surface-duo2.dts @@ -358,8 +358,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p88>; + vdda-phy-supply = <&vreg_l1b_0p88>; + vdda-pll-supply = <&vreg_l6b_1p2>; }; &usb_2 { @@ -377,6 +377,6 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l5b_0p88>; + vdda-phy-supply = <&vreg_l5b_0p88>; + vdda-pll-supply = <&vreg_l6b_1p2>; }; diff --git a/arch/arm64/boot/dts/qcom/sm8350-mtp.dts b/arch/arm64/boot/dts/qcom/sm8350-mtp.dts index 8bee57f3b25a19..859985e561aa52 100644 --- a/arch/arm64/boot/dts/qcom/sm8350-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sm8350-mtp.dts @@ -354,8 +354,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p88>; + vdda-phy-supply = <&vreg_l1b_0p88>; + vdda-pll-supply = <&vreg_l6b_1p2>; }; &usb_2 { @@ -373,8 +373,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l5b_0p88>; + vdda-phy-supply = <&vreg_l5b_0p88>; + vdda-pll-supply = <&vreg_l6b_1p2>; }; &ipa { diff --git a/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi b/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi index 7ae1eb0a7ccea4..1f9d094fee8fc8 100644 --- a/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi @@ -917,6 +917,6 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&pm8350_l6>; - vdda-pll-supply = <&pm8350_l1>; + vdda-phy-supply = <&pm8350_l1>; + vdda-pll-supply = <&pm8350_l6>; }; diff --git a/arch/arm64/boot/dts/qcom/sm8350.dtsi b/arch/arm64/boot/dts/qcom/sm8350.dtsi index c830953156ec69..20c3ab9465d2ec 100644 --- a/arch/arm64/boot/dts/qcom/sm8350.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8350.dtsi @@ -1575,8 +1575,8 @@ "aggre1", "aggre0"; - iommu-map = <0x0 &apps_smmu 0x1c00 0x1>, - <0x100 &apps_smmu 0x1c01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c00 0x0 0x1>, + <0x100 &apps_smmu 0x1c01 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -1684,8 +1684,8 @@ "ddrss_sf_tbu", "aggre1"; - iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, - <0x100 &apps_smmu 0x1c81 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c80 0x0 0x1>, + <0x100 &apps_smmu 0x1c81 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>; reset-names = "pci"; diff --git a/arch/arm64/boot/dts/qcom/sm8450-hdk.dts b/arch/arm64/boot/dts/qcom/sm8450-hdk.dts index b37998cd9a2c35..4ade28f40cae08 100644 --- a/arch/arm64/boot/dts/qcom/sm8450-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8450-hdk.dts @@ -1214,8 +1214,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p91>; + vdda-phy-supply = <&vreg_l1b_0p91>; + vdda-pll-supply = <&vreg_l6b_1p2>; }; &usb_1_qmpphy_out { diff --git a/arch/arm64/boot/dts/qcom/sm8450-qrd.dts b/arch/arm64/boot/dts/qcom/sm8450-qrd.dts index 56db5f79f59d29..4993059483e9ab 100644 --- a/arch/arm64/boot/dts/qcom/sm8450-qrd.dts +++ b/arch/arm64/boot/dts/qcom/sm8450-qrd.dts @@ -527,8 +527,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p91>; + vdda-phy-supply = <&vreg_l1b_0p91>; + vdda-pll-supply = <&vreg_l6b_1p2>; }; &usb_1_qmpphy_out { diff --git a/arch/arm64/boot/dts/qcom/sm8450-sony-xperia-nagara.dtsi b/arch/arm64/boot/dts/qcom/sm8450-sony-xperia-nagara.dtsi index 6bd315e1099253..4fb5627d812641 100644 --- a/arch/arm64/boot/dts/qcom/sm8450-sony-xperia-nagara.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8450-sony-xperia-nagara.dtsi @@ -793,7 +793,7 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&pm8350_l6>; - vdda-pll-supply = <&pm8350_l1>; + vdda-phy-supply = <&pm8350_l1>; + vdda-pll-supply = <&pm8350_l6>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8450.dtsi b/arch/arm64/boot/dts/qcom/sm8450.dtsi index 56cb6e959e4ee2..18ab94de4ac9ec 100644 --- a/arch/arm64/boot/dts/qcom/sm8450.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8450.dtsi @@ -2023,8 +2023,8 @@ "aggre0", "aggre1"; - iommu-map = <0x0 &apps_smmu 0x1c00 0x1>, - <0x100 &apps_smmu 0x1c01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c00 0x0 0x1>, + <0x100 &apps_smmu 0x1c01 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -2188,8 +2188,8 @@ "ddrss_sf_tbu", "aggre1"; - iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, - <0x100 &apps_smmu 0x1c81 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c80 0x0 0x1>, + <0x100 &apps_smmu 0x1c81 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>; reset-names = "pci"; diff --git a/arch/arm64/boot/dts/qcom/sm8550-hdk.dts b/arch/arm64/boot/dts/qcom/sm8550-hdk.dts index ee13e6136a8259..25ff5e39dd3465 100644 --- a/arch/arm64/boot/dts/qcom/sm8550-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8550-hdk.dts @@ -1374,8 +1374,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3f_0p88>; + vdda-phy-supply = <&vreg_l3f_0p88>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8550-mtp.dts b/arch/arm64/boot/dts/qcom/sm8550-mtp.dts index 5769be83cfbd3b..7c634572c6b715 100644 --- a/arch/arm64/boot/dts/qcom/sm8550-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sm8550-mtp.dts @@ -971,8 +971,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3f_0p91>; + vdda-phy-supply = <&vreg_l3f_0p91>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8550-qrd.dts b/arch/arm64/boot/dts/qcom/sm8550-qrd.dts index 2fb2e0be5e4c6b..ceb5cd3c5bf3e8 100644 --- a/arch/arm64/boot/dts/qcom/sm8550-qrd.dts +++ b/arch/arm64/boot/dts/qcom/sm8550-qrd.dts @@ -1274,8 +1274,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3f_0p88>; + vdda-phy-supply = <&vreg_l3f_0p88>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8550-sony-xperia-yodo-pdx234.dts b/arch/arm64/boot/dts/qcom/sm8550-sony-xperia-yodo-pdx234.dts index 0e6ed6fce61470..367f6eae5e6529 100644 --- a/arch/arm64/boot/dts/qcom/sm8550-sony-xperia-yodo-pdx234.dts +++ b/arch/arm64/boot/dts/qcom/sm8550-sony-xperia-yodo-pdx234.dts @@ -766,8 +766,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&pm8550vs_2_l3>; - vdda-pll-supply = <&pm8550ve_l3>; + vdda-phy-supply = <&pm8550ve_l3>; + vdda-pll-supply = <&pm8550vs_2_l3>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8550.dtsi b/arch/arm64/boot/dts/qcom/sm8550.dtsi index 396201905ef25c..69babd26c6789c 100644 --- a/arch/arm64/boot/dts/qcom/sm8550.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8550.dtsi @@ -2382,8 +2382,8 @@ msi-map = <0x0 &gic_its 0x1400 0x1>, <0x100 &gic_its 0x1401 0x1>; - iommu-map = <0x0 &apps_smmu 0x1400 0x1>, - <0x100 &apps_smmu 0x1401 0x1>; + iommu-map = <0x0 &apps_smmu 0x1400 0x0 0x1>, + <0x100 &apps_smmu 0x1401 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -2561,8 +2561,8 @@ msi-map = <0x0 &gic_its 0x1480 0x1>, <0x100 &gic_its 0x1481 0x1>; - iommu-map = <0x0 &apps_smmu 0x1480 0x1>, - <0x100 &apps_smmu 0x1481 0x1>; + iommu-map = <0x0 &apps_smmu 0x1480 0x0 0x1>, + <0x100 &apps_smmu 0x1481 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>, <&gcc GCC_PCIE_1_LINK_DOWN_BCR>; diff --git a/arch/arm64/boot/dts/qcom/sm8650-ayaneo-pocket-s2.dts b/arch/arm64/boot/dts/qcom/sm8650-ayaneo-pocket-s2.dts index 0dc994f4e48d9f..6ddf30ecabfef3 100644 --- a/arch/arm64/boot/dts/qcom/sm8650-ayaneo-pocket-s2.dts +++ b/arch/arm64/boot/dts/qcom/sm8650-ayaneo-pocket-s2.dts @@ -1536,8 +1536,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3i_1p2>; - vdda-pll-supply = <&vreg_l3g_0p91>; + vdda-phy-supply = <&vreg_l3g_0p91>; + vdda-pll-supply = <&vreg_l3i_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8650-hdk.dts b/arch/arm64/boot/dts/qcom/sm8650-hdk.dts index eabc828c05b4c5..865ee15be9e21c 100644 --- a/arch/arm64/boot/dts/qcom/sm8650-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8650-hdk.dts @@ -1338,8 +1338,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3i_1p2>; - vdda-pll-supply = <&vreg_l3g_0p91>; + vdda-phy-supply = <&vreg_l3g_0p91>; + vdda-pll-supply = <&vreg_l3i_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8650-mtp.dts b/arch/arm64/boot/dts/qcom/sm8650-mtp.dts index dd6e33d2dc5d71..26045430d34ada 100644 --- a/arch/arm64/boot/dts/qcom/sm8650-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sm8650-mtp.dts @@ -886,8 +886,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3i_1p2>; - vdda-pll-supply = <&vreg_l3g_0p91>; + vdda-phy-supply = <&vreg_l3g_0p91>; + vdda-pll-supply = <&vreg_l3i_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8650-qrd.dts b/arch/arm64/boot/dts/qcom/sm8650-qrd.dts index a3982ae2292900..c00e3ae208cd8b 100644 --- a/arch/arm64/boot/dts/qcom/sm8650-qrd.dts +++ b/arch/arm64/boot/dts/qcom/sm8650-qrd.dts @@ -1357,8 +1357,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3i_1p2>; - vdda-pll-supply = <&vreg_l3g_0p91>; + vdda-phy-supply = <&vreg_l3g_0p91>; + vdda-pll-supply = <&vreg_l3i_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8650.dtsi b/arch/arm64/boot/dts/qcom/sm8650.dtsi index 65c4a2b46d2159..b01993ba82630b 100644 --- a/arch/arm64/boot/dts/qcom/sm8650.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8650.dtsi @@ -3626,8 +3626,8 @@ operating-points-v2 = <&pcie0_opp_table>; - iommu-map = <0 &apps_smmu 0x1400 0x1>, - <0x100 &apps_smmu 0x1401 0x1>; + iommu-map = <0 &apps_smmu 0x1400 0x0 0x1>, + <0x100 &apps_smmu 0x1401 0x0 0x1>; interrupt-map = <0 0 0 1 &intc 0 0 GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH 0>, <0 0 0 2 &intc 0 0 GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH 0>, @@ -3819,8 +3819,8 @@ operating-points-v2 = <&pcie1_opp_table>; - iommu-map = <0 &apps_smmu 0x1480 0x1>, - <0x100 &apps_smmu 0x1481 0x1>; + iommu-map = <0 &apps_smmu 0x1480 0x0 0x1>, + <0x100 &apps_smmu 0x1481 0x0 0x1>; interrupt-map = <0 0 0 1 &intc 0 0 GIC_SPI 434 IRQ_TYPE_LEVEL_HIGH 0>, <0 0 0 2 &intc 0 0 GIC_SPI 435 IRQ_TYPE_LEVEL_HIGH 0>, diff --git a/arch/arm64/boot/dts/qcom/sm8750-mtp.dts b/arch/arm64/boot/dts/qcom/sm8750-mtp.dts index 3837f6785320df..9b5fbc004df4ec 100644 --- a/arch/arm64/boot/dts/qcom/sm8750-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sm8750-mtp.dts @@ -1404,8 +1404,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3g_1p2>; - vdda-pll-supply = <&vreg_l2d_0p88>; + vdda-phy-supply = <&vreg_l2d_0p88>; + vdda-pll-supply = <&vreg_l3g_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8750-qrd.dts b/arch/arm64/boot/dts/qcom/sm8750-qrd.dts index 801c46d556022b..b56317cfa9fc7f 100644 --- a/arch/arm64/boot/dts/qcom/sm8750-qrd.dts +++ b/arch/arm64/boot/dts/qcom/sm8750-qrd.dts @@ -1132,8 +1132,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3g_1p2>; - vdda-pll-supply = <&vreg_l2d_0p88>; + vdda-phy-supply = <&vreg_l2d_0p88>; + vdda-pll-supply = <&vreg_l3g_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8750.dtsi b/arch/arm64/boot/dts/qcom/sm8750.dtsi index fafed417c66fc2..c15ad5de0aa87a 100644 --- a/arch/arm64/boot/dts/qcom/sm8750.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8750.dtsi @@ -5280,8 +5280,8 @@ interconnect-names = "pcie-mem", "cpu-pcie"; - iommu-map = <0x0 &apps_smmu 0x1400 0x1>, - <0x100 &apps_smmu 0x1401 0x1>; + iommu-map = <0x0 &apps_smmu 0x1400 0x0 0x1>, + <0x100 &apps_smmu 0x1401 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -5445,6 +5445,7 @@ phy-names = "ufsphy"; #reset-cells = <1>; + qcom,ice = <&ice>; status = "disabled"; diff --git a/arch/arm64/boot/dts/qcom/talos-evk-som.dtsi b/arch/arm64/boot/dts/qcom/talos-evk-som.dtsi index 294354c034c37c..6469c6fa9f634d 100644 --- a/arch/arm64/boot/dts/qcom/talos-evk-som.dtsi +++ b/arch/arm64/boot/dts/qcom/talos-evk-som.dtsi @@ -589,8 +589,8 @@ }; &usb_qmpphy_2 { - vdda-phy-supply = <&vreg_l11a>; - vdda-pll-supply = <&vreg_l5a>; + vdda-phy-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l11a>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/talos.dtsi b/arch/arm64/boot/dts/qcom/talos.dtsi index fb1bbc51bb8a45..23990890280bda 100644 --- a/arch/arm64/boot/dts/qcom/talos.dtsi +++ b/arch/arm64/boot/dts/qcom/talos.dtsi @@ -351,7 +351,7 @@ }; opp-1094400000 { - opp-hz = /bits/ 64 <109440000>; + opp-hz = /bits/ 64 <1094400000>; opp-peak-kBps = <(1017600 * 4) (940800 * 16)>; }; @@ -1349,8 +1349,8 @@ &config_noc SLAVE_PCIE_0 QCOM_ICC_TAG_ACTIVE_ONLY>; interconnect-names = "pcie-mem", "cpu-pcie"; - iommu-map = <0x0 &apps_smmu 0x400 0x1>, - <0x100 &apps_smmu 0x401 0x1>; + iommu-map = <0x0 &apps_smmu 0x400 0x0 0x1>, + <0x100 &apps_smmu 0x401 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -2017,7 +2017,7 @@ }; }; - gmu: gmu@506a000 { + gmu: gmu@506d000 { compatible = "qcom,adreno-rgmu-612.0", "qcom,adreno-rgmu"; reg = <0x0 0x0506d000 0x0 0x2c000>; diff --git a/arch/arm64/boot/dts/qcom/x1-asus-vivobook-s15.dtsi b/arch/arm64/boot/dts/qcom/x1-asus-vivobook-s15.dtsi index 48c4ad648354ea..0dc7c1b5a14a0b 100644 --- a/arch/arm64/boot/dts/qcom/x1-asus-vivobook-s15.dtsi +++ b/arch/arm64/boot/dts/qcom/x1-asus-vivobook-s15.dtsi @@ -1226,8 +1226,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1256,8 +1256,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1277,8 +1277,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; /delete-property/ mode-switch; /delete-property/ orientation-switch; @@ -1342,15 +1342,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi b/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi index 66d566808f5836..59cbbfbce4905d 100644 --- a/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi +++ b/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi @@ -1454,8 +1454,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p9>; + vdda-phy-supply = <&vreg_l1j_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1484,8 +1484,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1505,8 +1505,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; /delete-property/ mode-switch; /delete-property/ orientation-switch; @@ -1551,15 +1551,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p9>; + vdda-phy-supply = <&vreg_l3c_0p9>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p9>; + vdda-phy-supply = <&vreg_l3c_0p9>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1-crd.dtsi b/arch/arm64/boot/dts/qcom/x1-crd.dtsi index a9c5c523575eec..107fe2867e7366 100644 --- a/arch/arm64/boot/dts/qcom/x1-crd.dtsi +++ b/arch/arm64/boot/dts/qcom/x1-crd.dtsi @@ -1734,8 +1734,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1764,8 +1764,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1794,8 +1794,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1836,15 +1836,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi b/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi index d6de4da02dcd38..4cda708455d842 100644 --- a/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi +++ b/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi @@ -1570,8 +1570,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1600,8 +1600,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1639,8 +1639,8 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; @@ -1653,8 +1653,8 @@ }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi b/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi index 02708f23a86579..dba8cee43e2947 100644 --- a/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi +++ b/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi @@ -1466,8 +1466,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; @@ -1496,8 +1496,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; @@ -1533,8 +1533,8 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1-microsoft-denali.dtsi b/arch/arm64/boot/dts/qcom/x1-microsoft-denali.dtsi index 7559557610ed40..ce7d958ca68798 100644 --- a/arch/arm64/boot/dts/qcom/x1-microsoft-denali.dtsi +++ b/arch/arm64/boot/dts/qcom/x1-microsoft-denali.dtsi @@ -1268,8 +1268,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1298,8 +1298,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts b/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts index 2e38402e2c140d..be1a8ac8c699ee 100644 --- a/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts +++ b/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts @@ -1382,8 +1382,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1413,8 +1413,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1443,8 +1443,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1486,15 +1486,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi b/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi index 5d49df41be02a5..0bee2a3d6e7a9c 100644 --- a/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi +++ b/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi @@ -1601,8 +1601,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1631,8 +1631,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1652,8 +1652,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; /delete-property/ mode-switch; /delete-property/ orientation-switch; @@ -1717,15 +1717,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts b/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts index ce7b10ea89b6dc..2be5a37c00ccce 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts @@ -1286,8 +1286,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p9>; + vdda-phy-supply = <&vreg_l1j_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1316,8 +1316,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1357,15 +1357,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p9>; + vdda-phy-supply = <&vreg_l3c_0p9>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p9>; + vdda-phy-supply = <&vreg_l3c_0p9>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts b/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts index beb1475d7fa0c7..9eb2dfd99f20d4 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts @@ -1564,8 +1564,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1594,8 +1594,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1624,8 +1624,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-medion-sprchrgd-14-s1.dts b/arch/arm64/boot/dts/qcom/x1e80100-medion-sprchrgd-14-s1.dts index f95b1f9f439d36..e4a0b86847cdde 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-medion-sprchrgd-14-s1.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-medion-sprchrgd-14-s1.dts @@ -1407,8 +1407,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p9>; + vdda-phy-supply = <&vreg_l1j_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1428,8 +1428,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; /delete-property/ mode-switch; /delete-property/ orientation-switch; @@ -1497,16 +1497,16 @@ /* Right side USB-A (USB-SS 3) */ &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p9>; + vdda-phy-supply = <&vreg_l3c_0p9>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; /* Left side USB-A (USB-SS 4) */ &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p9>; + vdda-phy-supply = <&vreg_l3c_0p9>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi b/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi index 28342cb84ded76..13d0f09e0c2a1b 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi +++ b/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi @@ -1491,8 +1491,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j>; - vdda-pll-supply = <&vreg_l1j>; + vdda-phy-supply = <&vreg_l1j>; + vdda-pll-supply = <&vreg_l2j>; status = "okay"; }; @@ -1521,8 +1521,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j>; - vdda-pll-supply = <&vreg_l2d>; + vdda-phy-supply = <&vreg_l2d>; + vdda-pll-supply = <&vreg_l2j>; status = "okay"; }; @@ -1565,15 +1565,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e>; - vdda-pll-supply = <&vreg_l3c>; + vdda-phy-supply = <&vreg_l3c>; + vdda-pll-supply = <&vreg_l3e>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e>; - vdda-pll-supply = <&vreg_l3c>; + vdda-phy-supply = <&vreg_l3c>; + vdda-pll-supply = <&vreg_l3e>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts b/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts index 8afbac349cc910..78380fec8839eb 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts @@ -1416,8 +1416,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1446,8 +1446,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1476,8 +1476,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1519,15 +1519,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts b/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts index 500809772097e2..913645bc3595d3 100644 --- a/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts +++ b/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts @@ -1561,8 +1561,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1591,8 +1591,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1631,8 +1631,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; /delete-property/ mode-switch; /delete-property/ orientation-switch; @@ -1685,8 +1685,8 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; @@ -1701,8 +1701,8 @@ }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/renesas/r9a09g047.dtsi b/arch/arm64/boot/dts/renesas/r9a09g047.dtsi index b48da8534a3dfb..b6193c15837063 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g047.dtsi +++ b/arch/arm64/boot/dts/renesas/r9a09g047.dtsi @@ -931,7 +931,7 @@ reg = <0 0x13400000 0 0x10000>; ranges = <0x02000000 0 0x30000000 0 0x30000000 0 0x08000000>, <0x43000000 4 0x40000000 4 0x40000000 6 0x00000000>; - dma-ranges = <0x42000000 0 0x40000000 0 0x40000000 2 0x00000000>; + dma-ranges = <0x43000000 0 0x40000000 0 0x40000000 2 0x00000000>; bus-range = <0x0 0xff>; interrupts = , , diff --git a/arch/arm64/boot/dts/renesas/r9a09g056.dtsi b/arch/arm64/boot/dts/renesas/r9a09g056.dtsi index d6c8c39df2a4d9..5a3a6f72029a26 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g056.dtsi +++ b/arch/arm64/boot/dts/renesas/r9a09g056.dtsi @@ -1049,7 +1049,7 @@ reg = <0 0x13400000 0 0x10000>; ranges = <0x02000000 0 0x30000000 0 0x30000000 0 0x8000000>, <0x43000000 4 0x40000000 4 0x40000000 6 0x00000000>; - dma-ranges = <0x42000000 0 0x40000000 0 0x40000000 2 0x00000000>; + dma-ranges = <0x43000000 0 0x40000000 0 0x40000000 2 0x00000000>; bus-range = <0x0 0xff>; interrupts = , , diff --git a/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts b/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts index e9ed2de128f6f2..987e44d0bf9576 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts +++ b/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts @@ -256,8 +256,7 @@ /* * GMAC1 Pin Configuration: * - * SW2[8] ON - use pins P33_2-P33_7, P34_0-P34_5, P34_7 and - * P35_0-P35_2 for Ethernet port 3 + * SW2[8] ON - use pins P33_2-P33_7 and P34_0-P34_5 for Ethernet port 3 */ gmac1_pins: gmac1-pins { pinmux = , /* ETH3_TXCLK */ @@ -272,10 +271,6 @@ , /* ETH3_RXD2 */ , /* ETH3_RXD3 */ , /* ETH3_RXDV */ - , /* ETH3_TXER */ - , /* ETH3_RXER */ - , /* ETH3_CRS */ - , /* ETH3_COL */ , /* GMAC1_MDC */ , /* GMAC1_MDIO */ , /* ETH3_REFCLK */ @@ -286,8 +281,7 @@ * GMAC2 Pin Configuration: * * SW2[6] OFF - connect MDC/MDIO of Ethernet port 2 to GMAC2 - * SW2[7] ON - use pins P29_1-P29_7, P30_0-P30_4, and P31_2-P31_5 - * for Ethernet port 2 + * SW2[7] ON - use pins P29_1-P29_7 and P30_0-P30_4 for Ethernet port 2 */ gmac2_pins: gmac2-pins { pinmux = , /* ETH2_TXCLK */ @@ -302,10 +296,6 @@ , /* ETH2_RXD2 */ , /* ETH2_RXD3 */ , /* ETH2_RXDV */ - , /* ETH2_TXER */ - , /* ETH2_RXER */ - , /* ETH2_CRS */ - , /* ETH2_COL */ , /* GMAC2_MDC */ , /* GMAC2_MDIO */ , /* ETH2_REFCLK */ diff --git a/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts b/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts index ef6cc7497c2c4c..a66502d8d82bdd 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts +++ b/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts @@ -339,9 +339,7 @@ /* * GMAC1 Pin Configuration: * - * DSW5[8] ON - use pins P00_0-P00_2, P33_2-P33_7, P34_0-P34_6 - * for Ethernet port 3 - * DSW12[1] OFF; DSW12[2] ON - use pin P00_3 for Ethernet port 3 + * DSW5[8] ON - use pins P33_2-P33_7 and P34_0-P34_6 for Ethernet port 3 */ gmac1_pins: gmac1-pins { pinmux = , /* ETH3_TXCLK */ @@ -356,10 +354,6 @@ , /* ETH3_RXD2 */ , /* ETH3_RXD3 */ , /* ETH3_RXDV */ - , /* ETH3_TXER */ - , /* ETH3_RXER */ - , /* ETH3_CRS */ - , /* ETH3_COL */ , /* GMAC1_MDC */ , /* GMAC1_MDIO */ , /* ETH3_REFCLK */ @@ -370,8 +364,7 @@ * GMAC2 Pin Configuration: * * DSW5[6] OFF - connect MDC/MDIO of Ethernet port 2 to GMAC2 - * DSW5[7] ON - use pins P29_1-P29_7, P30_0-P30_4, P30_7, - * P31_2, P31_4 and P31_5 are used for Ethernet port 2 + * DSW5[7] ON - use pins P29_1-P29_7 and P30_0-P30_4 for Ethernet port 2 * DSW13[7] OFF; DSW13[8] ON - use pin P13_7 for IRQ14 */ gmac2_pins: gmac2-pins { @@ -387,10 +380,6 @@ , /* ETH2_RXD2 */ , /* ETH2_RXD3 */ , /* ETH2_RXDV */ - , /* ETH2_TXER */ - , /* ETH2_RXER */ - , /* ETH2_CRS */ - , /* ETH2_COL */ , /* GMAC2_MDC */ , /* GMAC2_MDIO */ , /* ETH2_REFCLK */ diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi index 3f3cb0eb58096e..5435fbc270954c 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi @@ -505,8 +505,7 @@ ap_i2c_tp: &i2c5 { mvl_wifi: wifi@0,0 { compatible = "pci1b4b,2b42"; reg = <0x0000 0x0 0x0 0x0 0x0>; - interrupt-parent = <&gpio0>; - interrupts = <8 IRQ_TYPE_LEVEL_LOW>; + interrupts-extended = <&gpio0 8 IRQ_TYPE_LEVEL_LOW>; pinctrl-names = "default"; pinctrl-0 = <&wlan_host_wake_l>; wakeup-source; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-bigtreetech-cb2.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-bigtreetech-cb2.dtsi index b6cf03a7ba66bc..04cf285e6c2afb 100644 --- a/arch/arm64/boot/dts/rockchip/rk3566-bigtreetech-cb2.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3566-bigtreetech-cb2.dtsi @@ -569,7 +569,7 @@ reg = <0x48>; status = "okay"; ti,x-plate-ohms = <660>; - ti,rt-thr = <3000>; + ti,max-rt = <3000>; ti,fuzzx = <32>; ti,fuzzy = <16>; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi index fc1fdbfd316223..376ad04e07869c 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi @@ -1910,8 +1910,8 @@ edp0: edp@fdec0000 { compatible = "rockchip,rk3588-edp"; reg = <0x0 0xfdec0000 0x0 0x1000>; - clocks = <&cru CLK_EDP0_24M>, <&cru PCLK_EDP0>; - clock-names = "dp", "pclk"; + clocks = <&cru CLK_EDP0_24M>, <&cru PCLK_EDP0>, <&cru HCLK_VO1>; + clock-names = "dp", "pclk", "hclk"; interrupts = ; phys = <&hdptxphy0>; phy-names = "dp"; diff --git a/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi index a2640014ee0421..b251bb129cdbfc 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi @@ -285,8 +285,8 @@ edp1: edp@fded0000 { compatible = "rockchip,rk3588-edp"; reg = <0x0 0xfded0000 0x0 0x1000>; - clocks = <&cru CLK_EDP1_24M>, <&cru PCLK_EDP1>; - clock-names = "dp", "pclk"; + clocks = <&cru CLK_EDP1_24M>, <&cru PCLK_EDP1>, <&cru HCLK_VO1>; + clock-names = "dp", "pclk", "hclk"; interrupts = ; phys = <&hdptxphy1>; phy-names = "dp"; diff --git a/arch/arm64/boot/dts/ti/k3-am64-main.dtsi b/arch/arm64/boot/dts/ti/k3-am64-main.dtsi index 1b1d3970888b8a..a9a83781d40f54 100644 --- a/arch/arm64/boot/dts/ti/k3-am64-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am64-main.dtsi @@ -1401,7 +1401,7 @@ icssg0_mdio: mdio@32400 { compatible = "ti,davinci_mdio"; reg = <0x32400 0x100>; - clocks = <&k3_clks 62 3>; + clocks = <&k3_clks 81 0>; clock-names = "fck"; #address-cells = <1>; #size-cells = <0>; diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index c2a627f393144a..6dfbcacd9ba090 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -264,7 +264,7 @@ static inline void task_set_sve_vl_onexec(struct task_struct *task, #define SCTLR_USER_MASK \ (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | SCTLR_ELx_ENDA | SCTLR_ELx_ENDB | \ - SCTLR_EL1_TCF0_MASK) + SCTLR_EL1_TCF0_MASK | SCTLR_EL1_TCSO0_MASK) static inline void arch_thread_struct_whitelist(unsigned long *offset, unsigned long *size) diff --git a/arch/arm64/include/asm/ptdump.h b/arch/arm64/include/asm/ptdump.h index 50a195eda8ed22..3aa8fa2fcec763 100644 --- a/arch/arm64/include/asm/ptdump.h +++ b/arch/arm64/include/asm/ptdump.h @@ -50,6 +50,8 @@ struct ptdump_pg_state { const struct addr_marker *marker; const struct mm_struct *mm; unsigned long start_address; + /* exclusive end, ULONG_MAX represents an end at 1 << 64 */ + unsigned long end_address; int level; ptval_t current_prot; bool check_wx; diff --git a/arch/arm64/include/asm/rsi_cmds.h b/arch/arm64/include/asm/rsi_cmds.h index 2c8763876dfb77..c1fab41f671ec9 100644 --- a/arch/arm64/include/asm/rsi_cmds.h +++ b/arch/arm64/include/asm/rsi_cmds.h @@ -88,6 +88,14 @@ static inline long rsi_set_addr_range_state(phys_addr_t start, return res.a0; } +#define RSI_ATTEST_CHALLENGE_MIN_SIZE 32 +#define RSI_ATTEST_CHALLENGE_MAX_SIZE 64 + +struct rsi_attestation_token_init_args { + unsigned long fid; + u8 challenge[RSI_ATTEST_CHALLENGE_MAX_SIZE]; +}; + /** * rsi_attestation_token_init - Initialise the operation to retrieve an * attestation token. @@ -109,18 +117,21 @@ static inline long rsi_set_addr_range_state(phys_addr_t start, static inline long rsi_attestation_token_init(const u8 *challenge, unsigned long size) { - struct arm_smccc_1_2_regs regs = { 0 }; + union { + struct arm_smccc_1_2_regs regs; + struct rsi_attestation_token_init_args init; + } args = { 0 }; - /* The challenge must be at least 32bytes and at most 64bytes */ - if (!challenge || size < 32 || size > 64) + if (!challenge || size < RSI_ATTEST_CHALLENGE_MIN_SIZE || + size > RSI_ATTEST_CHALLENGE_MAX_SIZE) return -EINVAL; - regs.a0 = SMC_RSI_ATTESTATION_TOKEN_INIT; - memcpy(®s.a1, challenge, size); - arm_smccc_1_2_smc(®s, ®s); + args.init.fid = SMC_RSI_ATTESTATION_TOKEN_INIT; + memcpy(args.init.challenge, challenge, size); + arm_smccc_1_2_smc(&args.regs, &args.regs); - if (regs.a0 == RSI_SUCCESS) - return regs.a1; + if (args.regs.a0 == RSI_SUCCESS) + return args.regs.a1; return -EINVAL; } diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index 30cd7f8043986a..0ec90fd1754edc 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -184,6 +184,8 @@ void arch_efi_call_virt_setup(void) efi_virtmap_load(); } + __efi_fpsimd_begin(); + /* * Enable access to the valid TTBR0_EL1 and invoke the errata * workaround directly since there is no return from exception when @@ -191,8 +193,6 @@ void arch_efi_call_virt_setup(void) */ uaccess_ttbr0_enable(); post_ttbr_update_workaround(); - - __efi_fpsimd_begin(); } void arch_efi_call_virt_teardown(void) diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index ceb4eb11232a61..bda824628a20ea 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -52,16 +52,36 @@ static noinstr irqentry_state_t arm64_enter_from_kernel_mode(struct pt_regs *reg * After this function returns it is not safe to call regular kernel code, * instrumentable code, or any code which may trigger an exception. */ -static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs, - irqentry_state_t state) +static void noinstr __arm64_exit_to_kernel_mode(struct pt_regs *regs, + irqentry_state_t state) { - local_irq_disable(); - irqentry_exit_to_kernel_mode_preempt(regs, state); local_daif_mask(); mte_check_tfsr_exit(); irqentry_exit_to_kernel_mode_after_preempt(regs, state); } +/* + * We are returning from the context which allows involuntary kernel preemption + */ +static void noinstr arm64_exit_to_kernel_mode_preempt(struct pt_regs *regs, + irqentry_state_t state) +{ + irqentry_exit_to_kernel_mode_preempt(regs, state); + __arm64_exit_to_kernel_mode(regs, state); +} + +static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs, + irqentry_state_t state) +{ + if (!regs_irqs_disabled(regs)) { + local_irq_disable(); + arm64_exit_to_kernel_mode_preempt(regs, state); + return; + } + + __arm64_exit_to_kernel_mode(regs, state); +} + static __always_inline void arm64_syscall_enter_from_user_mode(struct pt_regs *regs) { enter_from_user_mode(regs); @@ -509,7 +529,7 @@ static __always_inline void __el1_irq(struct pt_regs *regs, do_interrupt_handler(regs, handler); irq_exit_rcu(); - arm64_exit_to_kernel_mode(regs, state); + arm64_exit_to_kernel_mode_preempt(regs, state); } static void noinstr el1_interrupt(struct pt_regs *regs, void (*handler)(struct pt_regs *)) diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 9717568518ba74..7bf1174277772e 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -348,8 +348,10 @@ int swsusp_arch_suspend(void) crash_prepare_suspend(); ret = swsusp_mte_save_tags(); - if (ret) + if (ret) { + local_daif_restore(flags); return ret; + } sleep_cpu = smp_processor_id(); ret = swsusp_save(); @@ -465,9 +467,21 @@ int __nocfi swsusp_arch_resume(void) if (el2_reset_needed()) __hyp_set_vectors(el2_vectors); + /* + * It is necessary to mask all DAIF exceptions here as: + * + * - The copy of swsusp_arch_suspend_exit() in the hibernation + * text cannot handle taking any exceptions. + * + * - The suspended kernel masked all DAIF exceptions in + * swsusp_arch_resume(), and expects to be re-entered in the + * same state : with all DAIF exceptions masked. + */ + local_daif_save(); hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1, resume_hdr.reenter_kernel, restore_pblist, resume_hdr.__hyp_stub_vectors, virt_to_phys(zero_page)); + unreachable(); return 0; } diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 4d08598e2891d3..4955166723b88a 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -560,6 +560,42 @@ static int gpr_get(struct task_struct *target, return membuf_write(&to, uregs, sizeof(*uregs)); } +static void update_syscall_orig_x0_after_ptrace(struct task_struct *target) +{ + struct pt_regs *regs = task_pt_regs(target); + struct kernel_siginfo *info = target->last_siginfo; + + /* + * Skip the update for NO_SYSCALL (set either by the user or the + * tracer), as regs[0] holds the return value (see the comment in + * el0_svc_common()) and can be unwound using syscall_rollback(). + */ + if (regs->syscallno == NO_SYSCALL) + return; + + /* We should only be called when target is in a ptrace stop */ + if (WARN_ON_ONCE(!info)) + return; + + /* + * For compat tasks, orig_r0 is provided directly through GPR index + * 17. + */ + if (is_compat_thread(task_thread_info(target))) + return; + + /* + * Don't update orig_x0 for a syscall-exit-stop, as x0 now contains the + * return value of the system call. + */ + if ((info->si_code & ~0x80) == SIGTRAP && + target->ptrace_message == PTRACE_EVENTMSG_SYSCALL_EXIT) { + return; + } + + regs->orig_x0 = regs->regs[0]; +} + static int gpr_set(struct task_struct *target, const struct user_regset *regset, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) @@ -575,6 +611,14 @@ static int gpr_set(struct task_struct *target, const struct user_regset *regset, return -EINVAL; task_pt_regs(target)->user_regs = newregs; + + /* + * Keep orig_x0 authoritative so that seccomp (via + * syscall_get_arguments()), audit and the restart path all see the same + * first argument the syscall is dispatched with, even if it has been + * updated by a tracer. + */ + update_syscall_orig_x0_after_ptrace(target); return 0; } @@ -753,6 +797,12 @@ static int system_call_set(struct task_struct *target, return ret; task_pt_regs(target)->syscallno = syscallno; + + /* + * Re-sync orig_x0 in case the syscall number has been changed + * from NO_SYSCALL. + */ + update_syscall_orig_x0_after_ptrace(target); return ret; } @@ -801,7 +851,7 @@ static void sve_init_header_from_task(struct user_sve_header *header, if (active) header->size = SVE_PT_SIZE(vq, header->flags); else - header->size = sizeof(header); + header->size = sizeof(*header); header->max_size = SVE_PT_SIZE(sve_vq_from_vl(header->max_vl), SVE_PT_REGS_SVE); } @@ -837,7 +887,7 @@ static int sve_get_common(struct task_struct *target, * from the other mode to userspace. */ if (header.size == sizeof(header)) - return 0; + return to.left; switch ((header.flags & SVE_PT_REGS_MASK)) { case SVE_PT_REGS_FPSIMD: diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index cdcdd160e5b696..3ab90aa24efb91 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -1086,7 +1086,7 @@ static void ipi_teardown(int cpu) disable_percpu_irq(ipi_irq_base + i); } } else { - disable_irq(irq_desc_get_irq(get_ipi_desc(cpu, i))); + disable_irq_nosync(irq_desc_get_irq(get_ipi_desc(cpu, i))); } } } diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index 448c917494f305..b33e1ca4a7815b 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -1,4 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 + +# KCSAN uses udelay for introducing watchpoint delay; avoid recursion. +KCSAN_SANITIZE_delay.o := n + lib-y := clear_user.o delay.o copy_from_user.o \ copy_to_user.o copy_page.o \ clear_page.o csum.o insn.o memchr.o memcpy.o \ diff --git a/arch/arm64/mm/ptdump.c b/arch/arm64/mm/ptdump.c index 5a76c59b5ada55..9d9bcb6738f4c8 100644 --- a/arch/arm64/mm/ptdump.c +++ b/arch/arm64/mm/ptdump.c @@ -278,9 +278,19 @@ void note_page_pgd(struct ptdump_state *pt_st, unsigned long addr, pgd_t pgd) void note_page_flush(struct ptdump_state *pt_st) { + struct ptdump_pg_state *st = container_of(pt_st, struct ptdump_pg_state, ptdump); + unsigned long end = st->end_address; pte_t pte_zero = {0}; - note_page(pt_st, 0, -1, pte_val(pte_zero)); + /* + * Address spaces that end at 1 << 64 have end_address == ULONG_MAX, + * but note_page() expects the exclusive end. In this case adjust end + * to the wraparound value 0. + */ + if (end == ULONG_MAX) + end = 0; + + note_page(pt_st, end, -1, pte_val(pte_zero)); } void ptdump_walk(struct seq_file *s, struct ptdump_info *info) @@ -296,6 +306,7 @@ void ptdump_walk(struct seq_file *s, struct ptdump_info *info) .marker = info->markers, .mm = info->mm, .pg_level = &kernel_pg_levels[0], + .end_address = end, .level = -1, .ptdump = { .note_page_pte = note_page_pte, @@ -337,6 +348,7 @@ bool ptdump_check_wx(void) { -1, NULL}, }, .pg_level = &kernel_pg_levels[0], + .end_address = ~0UL, .level = -1, .check_wx = true, .ptdump = { diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index b0075ece4a6e7f..d4e62484baea0e 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1082,23 +1082,27 @@ static void build_epilogue(struct jit_ctx *ctx, bool was_classic) * * Bit layout of `fixup` (32-bit): * - * +-----------+--------+-----------+-----------+----------+ - * | 31-27 | 26-22 | 21 | 20-16 | 15-0 | - * | | | | | | - * | FIXUP_REG | Unused | ARENA_ACC | ARENA_REG | OFFSET | - * +-----------+--------+-----------+-----------+----------+ + * +-----------+--------+-------------+-----------+-----------+----------+ + * | 31-27 | 26-23 | 22 | 21 | 20-16 | 15-0 | + * | | | | | | | + * | FIXUP_REG | Unused | ARENA_WRITE | ARENA_ACC | ARENA_REG | OFFSET | + * +-----------+--------+-------------+-----------+-----------+----------+ * * - OFFSET (16 bits): Offset used to compute address for Load/Store instruction. * - ARENA_REG (5 bits): Register that is used to calculate the address for load/store when * accessing the arena region. * - ARENA_ACCESS (1 bit): This bit is set when the faulting instruction accessed the arena region. + * - ARENA_WRITE (1 bit): This bit is set when the faulting instruction wrote to the arena region. + * It is independent of FIXUP_REG, since a read-modify-write both writes to + * memory and reads the old value into a register. * - FIXUP_REG (5 bits): Destination register for the load instruction (cleared on fault) or set to - * DONT_CLEAR if it is a store instruction. + * DONT_CLEAR if the instruction does not read into a register. */ #define BPF_FIXUP_OFFSET_MASK GENMASK(15, 0) #define BPF_FIXUP_ARENA_REG_MASK GENMASK(20, 16) #define BPF_ARENA_ACCESS BIT(21) +#define BPF_ARENA_WRITE BIT(22) #define BPF_FIXUP_REG_MASK GENMASK(31, 27) #define DONT_CLEAR 5 /* Unused ARM64 register from BPF's POV */ @@ -1109,7 +1113,7 @@ bool ex_handler_bpf(const struct exception_table_entry *ex, s16 off = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); int arena_reg = FIELD_GET(BPF_FIXUP_ARENA_REG_MASK, ex->fixup); bool is_arena = !!(ex->fixup & BPF_ARENA_ACCESS); - bool is_write = (dst_reg == DONT_CLEAR); + bool is_write = !!(ex->fixup & BPF_ARENA_WRITE); unsigned long addr; if (is_arena) { @@ -1132,7 +1136,7 @@ static int add_exception_handler(const struct bpf_insn *insn, { off_t ins_offset; s16 off = insn->off; - bool is_arena; + bool is_arena, is_write; int arena_reg; unsigned long pc; struct exception_table_entry *ex; @@ -1178,13 +1182,21 @@ static int add_exception_handler(const struct bpf_insn *insn, ex->insn = ins_offset; - if (BPF_CLASS(insn->code) != BPF_LDX) - dst_reg = DONT_CLEAR; + /* + * A load-acquire is of BPF_STX class, but reads from src_reg into + * dst_reg like a BPF_LDX does, hence it must not be treated as a store + * here. A read-modify-write carrying BPF_FETCH is reported as a write + * even though it does have a register to clear, see the callers. + */ + is_write = BPF_CLASS(insn->code) != BPF_LDX && + !bpf_atomic_is_load_acq(insn); ex->fixup = FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); if (is_arena) { ex->fixup |= BPF_ARENA_ACCESS; + if (is_write) + ex->fixup |= BPF_ARENA_WRITE; /* * insn->src_reg/dst_reg holds the address in the arena region with upper 32-bits * being zero because of a preceding addr_space_cast(r, 0x0, 0x1) instruction. @@ -1193,7 +1205,7 @@ static int add_exception_handler(const struct bpf_insn *insn, * memory access. Pass the reg holding the unmodified 32-bit address to * ex_handler_bpf. */ - if (BPF_CLASS(insn->code) == BPF_LDX) + if (BPF_CLASS(insn->code) == BPF_LDX || bpf_atomic_is_load_acq(insn)) arena_reg = bpf2a64[insn->src_reg]; else arena_reg = bpf2a64[insn->dst_reg]; @@ -1871,7 +1883,7 @@ emit_cond_jmp: break; } - ret = add_exception_handler(insn, ctx, dst); + ret = add_exception_handler(insn, ctx, DONT_CLEAR); if (ret) return ret; break; @@ -1938,7 +1950,7 @@ emit_cond_jmp: break; } - ret = add_exception_handler(insn, ctx, dst); + ret = add_exception_handler(insn, ctx, DONT_CLEAR); if (ret) return ret; break; @@ -1961,7 +1973,16 @@ emit_cond_jmp: return ret; if (BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) { - ret = add_exception_handler(insn, ctx, dst); + /* + * A load-acquire reads into dst_reg, and a read-modify-write + * carrying BPF_FETCH reads the old value into src_reg, or into + * r0 for a BPF_CMPXCHG. Clear that register on fault, the + * remaining atomics have no destination register. + */ + int load_reg = bpf_atomic_load_reg(insn); + + ret = add_exception_handler(insn, ctx, load_reg < 0 ? + DONT_CLEAR : bpf2a64[load_reg]); if (ret) return ret; } @@ -2491,9 +2512,8 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes) } static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, - const struct btf_func_model *m, - const struct arg_aux *a, - bool for_call_origin) + const struct btf_func_model *m, const struct arg_aux *a, + bool for_call_origin, bool is_struct_ops) { int i; int reg; @@ -2513,7 +2533,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, bargs_off += 8; } - soff = 32; /* on stack arguments start from FP + 32 */ + /* + * On-stack arguments start above the frame(s) pushed by the trampoline + * prologue. Entered through the fentry call from a traced function, the + * prologue saves both the parent (FP/x9) and the traced function + * (FP/LR) frames, so the arguments start at FP + 32. A struct_ops + * callback is called indirectly and only the FP/LR frame is saved, so + * they start at FP + 16. + */ + soff = is_struct_ops ? 16 : 32; doff = (for_call_origin ? oargs_off : bargs_off); /* save on stack arguments */ @@ -2709,7 +2737,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, store_func_meta(ctx, func_meta, func_meta_off); /* save args for bpf */ - save_args(ctx, bargs_off, oargs_off, m, a, false); + save_args(ctx, bargs_off, oargs_off, m, a, false, is_struct_ops); /* save callee saved registers */ emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx); @@ -2758,7 +2786,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, if (flags & BPF_TRAMP_F_CALL_ORIG) { /* save args for original func */ - save_args(ctx, bargs_off, oargs_off, m, a, true); + save_args(ctx, bargs_off, oargs_off, m, a, true, is_struct_ops); /* call original func */ emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx); emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx); diff --git a/arch/csky/kernel/entry.S b/arch/csky/kernel/entry.S index c68cdcc76d60e4..3261f46f224428 100644 --- a/arch/csky/kernel/entry.S +++ b/arch/csky/kernel/entry.S @@ -93,11 +93,11 @@ csky_syscall_trace: ldw a2, (sp, LSAVE_A2) ldw a3, (sp, LSAVE_A3) #if defined(__CSKYABIV2__) - subi sp, 8 ldw r9, (sp, LSAVE_A4) + ldw r10, (sp, LSAVE_A5) + subi sp, 8 stw r9, (sp, 0x0) - ldw r9, (sp, LSAVE_A5) - stw r9, (sp, 0x4) + stw r10, (sp, 0x4) jsr syscallid /* Do system call */ addi sp, 8 #else diff --git a/arch/m68k/emu/nfcon.c b/arch/m68k/emu/nfcon.c index d41260672e24b7..f4657d9f4f153d 100644 --- a/arch/m68k/emu/nfcon.c +++ b/arch/m68k/emu/nfcon.c @@ -49,7 +49,7 @@ static void nfcon_write(struct console *con, const char *str, static struct tty_driver *nfcon_device(struct console *con, int *index) { *index = 0; - return console_is_registered(con) ? nfcon_tty_driver : NULL; + return nfcon_tty_driver; } static struct console nf_console = { diff --git a/arch/m68k/kernel/traps.c b/arch/m68k/kernel/traps.c index d2f25e8895e5a5..60759ba3cbd45a 100644 --- a/arch/m68k/kernel/traps.c +++ b/arch/m68k/kernel/traps.c @@ -936,8 +936,8 @@ void show_stack(struct task_struct *task, unsigned long *stack, int i; if (!stack) { - if (task) - stack = (unsigned long *)task->thread.esp0; + if (task && task != current) + stack = (unsigned long *)task->thread.ksp; else stack = (unsigned long *)&stack; } diff --git a/arch/mips/kernel/ptrace.c b/arch/mips/kernel/ptrace.c index 3f4c94c8812417..87102a03b6eaff 100644 --- a/arch/mips/kernel/ptrace.c +++ b/arch/mips/kernel/ptrace.c @@ -1321,8 +1321,12 @@ long arch_ptrace(struct task_struct *child, long request, */ asmlinkage long syscall_trace_enter(struct pt_regs *regs) { + long syscall; + user_exit(); + syscall = current_thread_info()->syscall; + if (test_thread_flag(TIF_SYSCALL_TRACE)) { if (ptrace_report_syscall_entry(regs)) return -1; @@ -1342,7 +1346,7 @@ asmlinkage long syscall_trace_enter(struct pt_regs *regs) * Negative syscall numbers are mistaken for rejected syscalls, but * won't have had the return value set appropriately, so we do so now. */ - if (current_thread_info()->syscall < 0) + if (syscall < 0) syscall_set_return_value(current, regs, -ENOSYS, 0); return current_thread_info()->syscall; } diff --git a/arch/powerpc/configs/85xx-hw.config b/arch/powerpc/configs/85xx-hw.config index 2b19c20a9a2c48..6b847181014328 100644 --- a/arch/powerpc/configs/85xx-hw.config +++ b/arch/powerpc/configs/85xx-hw.config @@ -90,6 +90,7 @@ CONFIG_PPC_EPAPR_HV_BYTECHAN=y CONFIG_QE_GPIO=y CONFIG_QUICC_ENGINE=y CONFIG_RAPIDIO=y +CONFIG_RAS=y CONFIG_RTC_CLASS=y CONFIG_RTC_DRV_CMOS=y CONFIG_RTC_DRV_DS1307=y diff --git a/arch/powerpc/include/asm/syscall.h b/arch/powerpc/include/asm/syscall.h index 834fcc4f7b543b..19d1739af0b775 100644 --- a/arch/powerpc/include/asm/syscall.h +++ b/arch/powerpc/include/asm/syscall.h @@ -98,6 +98,12 @@ static inline void syscall_set_return_value(struct task_struct *task, regs->gpr[3] = val; } } + /* + * Mark that a return value has been explicitly set by seccomp or + * ptrace so that system_call_exception() can skip the syscall + * unconditionally, even when the user requested syscall(-1). + */ + set_thread_flag(TIF_SYSCALL_RET); } static inline void syscall_get_arguments(struct task_struct *task, diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h index 0487e94d341693..1e069a2e7ce875 100644 --- a/arch/powerpc/include/asm/thread_info.h +++ b/arch/powerpc/include/asm/thread_info.h @@ -120,6 +120,7 @@ void arch_setup_new_exec(void); #endif #define TIF_POLLING_NRFLAG 19 /* true if poll_idle() is polling TIF_NEED_RESCHED */ #define TIF_32BIT 20 /* 32 bit binary */ +#define TIF_SYSCALL_RET 21 /* syscall error value set */ /* as above, but as bit values */ #define _TIF_SYSCALL_TRACE (1<cause_ipi) + return; + smp_muxed_ipi_set_message(cpu, msg); /* diff --git a/arch/powerpc/kernel/syscall.c b/arch/powerpc/kernel/syscall.c index a9da2af6efa876..9d1b29f44ea0d7 100644 --- a/arch/powerpc/kernel/syscall.c +++ b/arch/powerpc/kernel/syscall.c @@ -22,6 +22,9 @@ notrace long system_call_exception(struct pt_regs *regs, unsigned long r0) add_random_kstack_offset(); r0 = syscall_enter_from_user_mode(regs, r0); + if (unlikely(test_and_clear_thread_flag(TIF_SYSCALL_RET))) + return syscall_get_error(current, regs); + if (unlikely(r0 >= NR_syscalls)) { if (unlikely(trap_is_unsupported_scv(regs))) { /* Unsupported scv vector */ diff --git a/arch/powerpc/kexec/crash.c b/arch/powerpc/kexec/crash.c index e6539f213b3d14..a520f851c3a6bb 100644 --- a/arch/powerpc/kexec/crash.c +++ b/arch/powerpc/kexec/crash.c @@ -502,7 +502,7 @@ static void update_crash_elfcorehdr(struct kimage *image, struct memory_notify * ret = get_crash_memory_ranges(&cmem); if (ret) { pr_err("Failed to get crash mem range\n"); - return; + goto out; } /* diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index dab106cae22b57..fc9db691e82037 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -471,8 +471,6 @@ static int bpf_jit_emit_func_call(u32 *image, struct codegen_context *ctx, u64 f int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context *ctx, u64 func) { unsigned long func_addr = func ? ppc_function_entry((void *)func) : 0; - long __maybe_unused reladdr; - int ret; /* bpf to bpf call, func is not known in the initial pass. Emit 5 nops as a placeholder */ if (!func) { @@ -487,6 +485,8 @@ int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context * } #ifdef CONFIG_PPC_KERNEL_PCREL + long reladdr; + reladdr = func_addr - local_paca->kernelbase; /* @@ -525,7 +525,7 @@ int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context * EMIT(PPC_RAW_BCTRL()); #else if (core_kernel_text(func_addr)) { - ret = bpf_jit_emit_func_call(image, ctx, func_addr, _R12); + int ret = bpf_jit_emit_func_call(image, ctx, func_addr, _R12); if (ret) return ret; } else { diff --git a/arch/powerpc/net/bpf_timed_may_goto.S b/arch/powerpc/net/bpf_timed_may_goto.S index 6fd8b1c9f4ac87..84ecf6fa7f5dcf 100644 --- a/arch/powerpc/net/bpf_timed_may_goto.S +++ b/arch/powerpc/net/bpf_timed_may_goto.S @@ -36,7 +36,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) * BPF_REG_FP is r31; BPF_REG_AX is r12 (stack offset in bytes). */ add r3, r31, r12 - bl bpf_check_timed_may_goto + bl CFUNC(bpf_check_timed_may_goto) /* Put return value back into AX */ mr r12, r3 diff --git a/arch/powerpc/platforms/44x/gpio.c b/arch/powerpc/platforms/44x/gpio.c index aea0d913b59d0b..4413a94cf7a6a6 100644 --- a/arch/powerpc/platforms/44x/gpio.c +++ b/arch/powerpc/platforms/44x/gpio.c @@ -169,6 +169,7 @@ static int ppc4xx_gpio_probe(struct platform_device *ofdev) gc = &chip->gc; + gc->parent = dev; gc->base = -1; gc->ngpio = 32; gc->direction_input = ppc4xx_gpio_dir_in; diff --git a/arch/powerpc/platforms/powernv/smp.c b/arch/powerpc/platforms/powernv/smp.c index 8f41ef364fc6f7..b1201dbafcaf6e 100644 --- a/arch/powerpc/platforms/powernv/smp.c +++ b/arch/powerpc/platforms/powernv/smp.c @@ -332,10 +332,12 @@ static void pnv_cause_ipi(int cpu) static void __init pnv_smp_probe(void) { - if (xive_enabled()) - xive_smp_probe(); - else + if (xive_enabled()) { + if (xive_smp_probe() < 0) + return; + } else { xics_smp_probe(); + } if (cpu_has_feature(CPU_FTR_DBELL)) { ic_cause_ipi = smp_ops->cause_ipi; diff --git a/arch/powerpc/platforms/pseries/smp.c b/arch/powerpc/platforms/pseries/smp.c index bf3d7ed3be010f..9e1bed383e3370 100644 --- a/arch/powerpc/platforms/pseries/smp.c +++ b/arch/powerpc/platforms/pseries/smp.c @@ -199,10 +199,12 @@ static int pseries_cause_nmi_ipi(int cpu) static __init void pSeries_smp_probe(void) { - if (xive_enabled()) - xive_smp_probe(); - else + if (xive_enabled()) { + if (xive_smp_probe() < 0) + return; + } else { xics_smp_probe(); + } /* No doorbell facility, must use the interrupt controller for IPIs */ if (!cpu_has_feature(CPU_FTR_DBELL)) diff --git a/arch/powerpc/sysdev/cpm_common.c b/arch/powerpc/sysdev/cpm_common.c index 07ea605ab0e628..b5d200e3ad684f 100644 --- a/arch/powerpc/sysdev/cpm_common.c +++ b/arch/powerpc/sysdev/cpm_common.c @@ -181,6 +181,18 @@ static int cpm2_gpio32_dir_in(struct gpio_chip *gc, unsigned int gpio) return 0; } +static int cpm2_gpio32_get_direction(struct gpio_chip *gc, unsigned int gpio) +{ + struct cpm2_gpio32_chip *cpm2_gc = gpiochip_get_data(gc); + struct cpm2_ioports __iomem *iop = cpm2_gc->regs; + u32 pin_mask = 1 << (31 - gpio); + + if (in_be32(&iop->dir) & pin_mask) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; +} + int cpm2_gpiochip_add32(struct device *dev) { struct device_node *np = dev->of_node; @@ -199,6 +211,7 @@ int cpm2_gpiochip_add32(struct device *dev) gc->ngpio = 32; gc->direction_input = cpm2_gpio32_dir_in; gc->direction_output = cpm2_gpio32_dir_out; + gc->get_direction = cpm2_gpio32_get_direction; gc->get = cpm2_gpio32_get; gc->set = cpm2_gpio32_set; gc->parent = dev; diff --git a/arch/powerpc/sysdev/xive/common.c b/arch/powerpc/sysdev/xive/common.c index dadd1f46ec9390..bbe7c85274ea97 100644 --- a/arch/powerpc/sysdev/xive/common.c +++ b/arch/powerpc/sysdev/xive/common.c @@ -74,6 +74,8 @@ static struct xive_ipi_desc { */ static unsigned int xive_ipi_cpu_to_irq(unsigned int cpu) { + if (!xive_ipis) + return XIVE_BAD_IRQ; return xive_ipis[early_cpu_to_node(cpu)].irq; } #endif @@ -1132,8 +1134,7 @@ static int __init xive_init_ipis(void) if (!ipi_domain) goto out_free_fwnode; - xive_ipis = kzalloc_objs(*xive_ipis, nr_node_ids, - GFP_KERNEL | __GFP_NOFAIL); + xive_ipis = kzalloc_objs(*xive_ipis, nr_node_ids, GFP_KERNEL); if (!xive_ipis) goto out_free_domain; @@ -1158,6 +1159,7 @@ static int __init xive_init_ipis(void) out_free_xive_ipis: kfree(xive_ipis); + xive_ipis = NULL; out_free_domain: irq_domain_remove(ipi_domain); out_free_fwnode: @@ -1190,6 +1192,9 @@ static int xive_setup_cpu_ipi(unsigned int cpu) pr_debug("Setting up IPI for CPU %d\n", cpu); + if (xive_ipi_irq == XIVE_BAD_IRQ) + return -EIO; + xc = per_cpu(xive_cpu, cpu); /* Check if we are already setup */ @@ -1234,6 +1239,9 @@ noinstr static void xive_cleanup_cpu_ipi(unsigned int cpu, struct xive_cpu *xc) /* Disable the IPI and free the IRQ data */ + if (xive_ipi_irq == XIVE_BAD_IRQ) + return; + /* Already cleaned up ? */ if (xc->hw_ipi == XIVE_BAD_IRQ) return; @@ -1257,15 +1265,19 @@ noinstr static void xive_cleanup_cpu_ipi(unsigned int cpu, struct xive_cpu *xc) xive_ops->put_ipi(cpu, xc); } -void __init xive_smp_probe(void) +int __init xive_smp_probe(void) { + int ret; + smp_ops->cause_ipi = xive_cause_ipi; /* Register the IPI */ - xive_init_ipis(); + ret = xive_init_ipis(); + if (ret < 0) + return ret; /* Allocate and setup IPI for the boot CPU */ - xive_setup_cpu_ipi(smp_processor_id()); + return xive_setup_cpu_ipi(smp_processor_id()); } #endif /* CONFIG_SMP */ diff --git a/arch/riscv/boot/dts/sophgo/cv180x.dtsi b/arch/riscv/boot/dts/sophgo/cv180x.dtsi index 06b0ce5a2db7af..ecafdee79ac8e0 100644 --- a/arch/riscv/boot/dts/sophgo/cv180x.dtsi +++ b/arch/riscv/boot/dts/sophgo/cv180x.dtsi @@ -417,7 +417,7 @@ }; dmac: dma-controller@4330000 { - compatible = "snps,axi-dma-1.01a"; + compatible = "sophgo,cv1800b-axi-dma", "snps,axi-dma-1.01a"; reg = <0x04330000 0x1000>; interrupts = ; clocks = <&clk CLK_SDMA_AXI>, <&clk CLK_SDMA_AXI>; diff --git a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts index d2abda5f5383aa..7f3dd08e74d795 100644 --- a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts +++ b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts @@ -101,7 +101,7 @@ phy-handle = <&rgmii0>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac0_cfg>; + pinctrl-0 = <&gmac0_cfg>, <&gmac0_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <0>; status = "okay"; @@ -124,7 +124,7 @@ phy-handle = <&rgmii1>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac1_cfg>; + pinctrl-0 = <&gmac1_cfg>, <&gmac1_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <250>; status = "okay"; diff --git a/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts b/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts index c800153077a80c..c76b91ecb914d6 100644 --- a/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts +++ b/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts @@ -121,7 +121,7 @@ phy-handle = <&rgmii0>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac0_cfg>; + pinctrl-0 = <&gmac0_cfg>, <&gmac0_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <0>; status = "okay"; @@ -144,7 +144,7 @@ phy-handle = <&rgmii1>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac1_cfg>; + pinctrl-0 = <&gmac1_cfg>, <&gmac1_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <250>; status = "okay"; diff --git a/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts b/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts index 96623454116eb7..246f8f2ab62b98 100644 --- a/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts +++ b/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts @@ -102,7 +102,7 @@ ð0 { phy-handle = <&rgmii0>; phy-mode = "rgmii-id"; - pinctrl-0 = <&gmac0_cfg>; + pinctrl-0 = <&gmac0_cfg>, <&gmac0_clk_ref_cfg>; pinctrl-names = "default"; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <0>; diff --git a/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts b/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts index 564a48c70b5d31..ae2fd8f7a85720 100644 --- a/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts +++ b/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts @@ -60,7 +60,7 @@ phy-handle = <&rgmii0>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac0_cfg>; + pinctrl-0 = <&gmac0_cfg>, <&gmac0_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <0>; status = "okay"; @@ -84,7 +84,7 @@ phy-handle = <&rgmii1>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac1_cfg>; + pinctrl-0 = <&gmac1_cfg>, <&gmac1_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <250>; status = "okay"; diff --git a/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi b/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi index 4e9a62d0e85b5d..8c57ca05dabdb4 100644 --- a/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi +++ b/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi @@ -27,8 +27,16 @@ , /* gmac0_tx_en */ , /* gmac0_mdc */ , /* gmac0_mdio */ - , /* gmac0_int_n */ - ; /* gmac0_clk_ref */ + ; /* gmac0_int_n */ + + bias-pull-up = <0>; + drive-strength = <21>; + }; + }; + + gmac0_clk_ref_cfg: gmac0-clk-ref-cfg { + gmac0-clk-ref-pins { + pinmux = ; /* gmac0_clk_ref */ bias-pull-up = <0>; drive-strength = <21>; @@ -51,8 +59,16 @@ , /* gmac1_tx_en */ , /* gmac1_mdc */ , /* gmac1_mdio */ - , /* gmac1_int_n */ - ; /* gmac1_clk_ref */ + ; /* gmac1_int_n */ + + bias-pull-up = <0>; + drive-strength = <21>; + }; + }; + + gmac1_clk_ref_cfg: gmac1-clk-ref-cfg { + gmac1-clk-ref-pins { + pinmux = ; /* gmac1_clk_ref */ bias-pull-up = <0>; drive-strength = <21>; diff --git a/arch/riscv/boot/dts/spacemit/k3-com260.dtsi b/arch/riscv/boot/dts/spacemit/k3-com260.dtsi index a38d7b738258de..c7a04a338d080c 100644 --- a/arch/riscv/boot/dts/spacemit/k3-com260.dtsi +++ b/arch/riscv/boot/dts/spacemit/k3-com260.dtsi @@ -173,12 +173,12 @@ status = "okay"; mdio { - phy1: phy@1 { - compatible = "ethernet-phy-ieee802.3-c22"; + phy1: ethernet-phy@1 { + compatible = "ethernet-phy-id001c.c916"; reg = <1>; reset-gpios = <&gpio 1 5 GPIO_ACTIVE_LOW>; reset-assert-us = <10000>; - reset-deassert-us = <10000>; + reset-deassert-us = <50000>; }; }; }; diff --git a/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts b/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts index b89c1521e6649b..86210cc6f77332 100644 --- a/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts +++ b/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts @@ -18,7 +18,7 @@ }; chosen { - stdout-path = "serial0"; + stdout-path = "serial0:115200n8"; }; memory@100000000 { @@ -190,12 +190,14 @@ status = "okay"; mdio { - phy0: phy@1 { - compatible = "ethernet-phy-ieee802.3-c22"; + phy0: ethernet-phy@1 { + compatible = "ethernet-phy-id001c.c916"; reg = <1>; reset-gpios = <&gpio 0 15 GPIO_ACTIVE_LOW>; reset-assert-us = <10000>; - reset-deassert-us = <10000>; + reset-deassert-us = <50000>; + realtek,aldps-enable; + realtek,clkout-disable; }; }; }; diff --git a/arch/riscv/boot/dts/thead/th1520.dtsi b/arch/riscv/boot/dts/thead/th1520.dtsi index 94932c51b7e392..db23624696b731 100644 --- a/arch/riscv/boot/dts/thead/th1520.dtsi +++ b/arch/riscv/boot/dts/thead/th1520.dtsi @@ -415,8 +415,7 @@ compatible = "thead,th1520-i2c", "snps,designware-i2c"; reg = <0xff 0xe7f24000 0x0 0x4000>; interrupts = <45 IRQ_TYPE_LEVEL_HIGH>; - clocks = <&clk CLK_I2C1>, <&clk CLK_PERI_APB_PCLK>; - clock-names = "ref", "pclk"; + clocks = <&clk CLK_I2C1>; #address-cells = <1>; #size-cells = <0>; status = "disabled"; diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index f46aa5602d74d3..64c7ec6abbf1fe 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -452,7 +452,8 @@ static const unsigned int riscv_c_exts[] = { /* * The canonical order of ISA extension names in the ISA string is defined in - * chapter 27 of the unprivileged specification. + * Chapter 27 of the RISC-V Instruction Set Manual Volume I Unprivileged ISA + * (Document Version 20191213). * * Ordinarily, for in-kernel data structures, this order is unimportant but * isa_ext_arr defines the order of the ISA string in /proc/cpuinfo. diff --git a/arch/riscv/kernel/machine_kexec_file.c b/arch/riscv/kernel/machine_kexec_file.c index 59d4bbc848a896..fa2946aa9b8f40 100644 --- a/arch/riscv/kernel/machine_kexec_file.c +++ b/arch/riscv/kernel/machine_kexec_file.c @@ -62,7 +62,7 @@ static int prepare_elf_headers(void **addr, unsigned long *sz) unsigned int nr_ranges; int ret; - nr_ranges = 1; /* For exclusion of crashkernel region */ + nr_ranges = 2; /* For exclusion of crashkernel region */ walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback); cmem = kmalloc_flex(*cmem, ranges, nr_ranges); @@ -77,8 +77,16 @@ static int prepare_elf_headers(void **addr, unsigned long *sz) /* Exclude crashkernel region */ ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end); - if (!ret) - ret = crash_prepare_elf64_headers(cmem, true, addr, sz); + if (ret) + goto out; + + if (crashk_low_res.end) { + ret = crash_exclude_mem_range(cmem, crashk_low_res.start, crashk_low_res.end); + if (ret) + goto out; + } + + ret = crash_prepare_elf64_headers(cmem, true, addr, sz); out: kfree(cmem); diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 0942116e59bc10..a3af5070ac66de 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -18,7 +18,6 @@ #define RV_MAX_REG_ARGS 8 #define RV_FENTRY_NINSNS 2 #define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4) -#define RV_KCFI_NINSNS (IS_ENABLED(CONFIG_CFI) ? 1 : 0) /* imm that allows emit_imm to emit max count insns */ #define RV_MAX_COUNT_IMM 0x7FFF7FF7FF7FF7FF @@ -272,8 +271,8 @@ static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) if (!is_tail_call) emit_addiw(RV_REG_A0, RV_REG_A5, 0, ctx); emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, - /* kcfi, fentry and TCC init insns will be skipped on tailcall */ - is_tail_call ? (RV_KCFI_NINSNS + RV_FENTRY_NINSNS + 1) * 4 : 0, + /* fentry and TCC init insns will be skipped on tailcall */ + is_tail_call ? (RV_FENTRY_NINSNS + 1) * 4 : 0, ctx); } @@ -1831,9 +1830,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, for (idx = 0; idx < fm->nr_args; idx++) { u8 reg = bpf_to_rv_reg(BPF_REG_1 + idx, ctx); + bool sign = fm->arg_flags[idx] & BTF_FMODEL_SIGNED_ARG; - if (fm->arg_size[idx] == sizeof(int)) - emit_sextw(reg, reg, ctx); + if (sign_extend(reg, reg, fm->arg_size[idx], sign, ctx)) + return -EINVAL; } } @@ -1986,7 +1986,21 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, else ret = emit_atomic_rmw(rd, rs, insn, ctx); - ret = ret ?: add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); + /* ret can be 1 (skip-zext); extable entry still needs to be added */ + if (ret >= 0) { + /* + * A load-acquire reads into dst_reg, and a read-modify-write + * carrying BPF_FETCH reads the old value into src_reg, or into + * r0 for a BPF_CMPXCHG. Clear that register on fault, the + * remaining atomics have no destination register. + */ + int load_reg = bpf_atomic_load_reg(insn); + + ret = add_exception_handler(insn, load_reg < 0 ? + REG_DONT_CLEAR_MARKER : regmap[load_reg], + ctx) ?: ret; + } + if (ret) return ret; break; @@ -2033,6 +2047,8 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) /* emit kcfi type preamble immediately before the first insn */ emit_kcfi(is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash, ctx); + /* bpf prog starts here as kcfi skipped during prog->bpf_func setup */ + /* nops reserved for auipc+jalr pair */ for (i = 0; i < RV_FENTRY_NINSNS; i++) emit(rv_nop(), ctx); diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index ce3bd3762e08cc..7cce1911861904 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -234,6 +234,7 @@ void bpf_jit_free(struct bpf_prog *prog) */ if (jit_data) { bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header); + kfree(jit_data->ctx.offset); kfree(jit_data); } hdr = bpf_jit_binary_pack_hdr(prog); diff --git a/arch/s390/include/asm/irqflags.h b/arch/s390/include/asm/irqflags.h index bcab456dfb803c..6eb6de6a99f2de 100644 --- a/arch/s390/include/asm/irqflags.h +++ b/arch/s390/include/asm/irqflags.h @@ -37,18 +37,24 @@ static __always_inline void __arch_local_irq_ssm(unsigned long flags) asm volatile("ssm %0" : : "Q" (flags) : "memory"); } -#ifdef CONFIG_KMSAN -#define arch_local_irq_attributes noinline notrace __no_sanitize_memory __maybe_unused +#if defined(CONFIG_KMSAN) && !defined(__DECOMPRESSOR) +unsigned long arch_local_save_flags(void); +unsigned long arch_local_irq_save(void); +void arch_local_irq_enable_external(void); +void arch_local_irq_enable(void); #else -#define arch_local_irq_attributes __always_inline +#define arch_local_save_flags __arch_local_save_flags +#define arch_local_irq_save __arch_local_irq_save +#define arch_local_irq_enable_external __arch_local_irq_enable_external +#define arch_local_irq_enable __arch_local_irq_enable #endif -static arch_local_irq_attributes unsigned long arch_local_save_flags(void) +static __always_inline unsigned long __arch_local_save_flags(void) { return __arch_local_irq_stnsm(0xff); } -static arch_local_irq_attributes unsigned long arch_local_irq_save(void) +static __always_inline unsigned long __arch_local_irq_save(void) { return __arch_local_irq_stnsm(0xfc); } @@ -58,12 +64,12 @@ static __always_inline void arch_local_irq_disable(void) arch_local_irq_save(); } -static arch_local_irq_attributes void arch_local_irq_enable_external(void) +static __always_inline void __arch_local_irq_enable_external(void) { __arch_local_irq_stosm(0x01); } -static arch_local_irq_attributes void arch_local_irq_enable(void) +static __always_inline void __arch_local_irq_enable(void) { __arch_local_irq_stosm(0x03); } diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 6c88476d79a311..14ef03cb2f72af 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -72,6 +72,7 @@ obj-$(CONFIG_STACKPROTECTOR) += stackprotector.o obj-$(CONFIG_KEXEC_FILE) += machine_kexec_file.o kexec_image.o obj-$(CONFIG_KEXEC_FILE) += kexec_elf.o obj-$(CONFIG_CERT_STORE) += cert_store.o +obj-$(CONFIG_KMSAN) += irqflags.o obj-$(CONFIG_PERF_EVENTS) += perf_event.o obj-$(CONFIG_PERF_EVENTS) += perf_cpum_cf.o perf_cpum_sf.o diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c index dbf430f479bdfe..14d2b58ad09304 100644 --- a/arch/s390/kernel/debug.c +++ b/arch/s390/kernel/debug.c @@ -993,8 +993,8 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) mutex_unlock(&debug_mutex); } -/* Remove debugfs entries and remove from internal list. */ -static void _debug_unregister(debug_info_t *id) +/* Remove debugfs entries. */ +static void _debug_unregister_debugfs(debug_info_t *id) { int i; @@ -1004,6 +1004,11 @@ static void _debug_unregister(debug_info_t *id) debugfs_remove(id->debugfs_entries[i]); } debugfs_remove(id->debugfs_root_entry); +} + +/* Remove from internal list. */ +static void _debug_unregister(debug_info_t *id) +{ if (id == debug_area_first) debug_area_first = id->next; if (id == debug_area_last) @@ -1029,6 +1034,7 @@ void debug_unregister(debug_info_t *id) mutex_lock(&debug_mutex); _debug_unregister(id); mutex_unlock(&debug_mutex); + _debug_unregister_debugfs(id); debug_info_put(id); } diff --git a/arch/s390/kernel/irqflags.c b/arch/s390/kernel/irqflags.c new file mode 100644 index 00000000000000..e192f59f8918c8 --- /dev/null +++ b/arch/s390/kernel/irqflags.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +noinstr unsigned long arch_local_save_flags(void) +{ + return __arch_local_save_flags(); +} +EXPORT_SYMBOL(arch_local_save_flags); + +noinstr unsigned long arch_local_irq_save(void) +{ + return __arch_local_irq_save(); +} +EXPORT_SYMBOL(arch_local_irq_save); + +noinstr void arch_local_irq_enable_external(void) +{ + __arch_local_irq_enable_external(); +} +EXPORT_SYMBOL(arch_local_irq_enable_external); + +noinstr void arch_local_irq_enable(void) +{ + __arch_local_irq_enable(); +} +EXPORT_SYMBOL(arch_local_irq_enable); diff --git a/arch/s390/kernel/vdso/Makefile b/arch/s390/kernel/vdso/Makefile index fece5d975eaf8f..35c834b895ecb3 100644 --- a/arch/s390/kernel/vdso/Makefile +++ b/arch/s390/kernel/vdso/Makefile @@ -30,7 +30,8 @@ KBUILD_CFLAGS_VDSO := $(filter-out -fno-asynchronous-unwind-tables,$(KBUILD_CFLA KBUILD_CFLAGS_VDSO += -fPIC -fno-common -fno-builtin -fasynchronous-unwind-tables KBUILD_CFLAGS_VDSO += -fno-stack-protector $(DISABLE_KSTACK_ERASE) ldflags-y := -shared -soname=linux-vdso.so.1 \ - --hash-style=both --build-id=sha1 -T + --hash-style=both --build-id=sha1 \ + $(call ld-option, --eh-frame-hdr) -T $(targets:%=$(obj)/%.dbg): KBUILD_CFLAGS = $(KBUILD_CFLAGS_VDSO) $(targets:%=$(obj)/%.dbg): KBUILD_AFLAGS = $(KBUILD_AFLAGS_VDSO) diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 31749c0362ca23..408ae09be3a59f 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -771,6 +771,8 @@ static void bpf_jit_probe_atomic_pre(struct bpf_jit *jit, struct bpf_insn *insn, struct bpf_jit_probe *probe) { + int load_reg; + if (BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) return; @@ -780,6 +782,14 @@ static void bpf_jit_probe_atomic_pre(struct bpf_jit *jit, EMIT4(0xb9080000, REG_W1, insn->dst_reg); probe->arena_reg = REG_W1; probe->prg = jit->prg; + /* + * A read-modify-write carrying BPF_FETCH reads the old value into + * src_reg, or into r0 for a BPF_CMPXCHG. Clear that register on + * fault, the remaining atomics only write memory. + */ + load_reg = bpf_atomic_load_reg(insn); + if (load_reg >= 0) + probe->reg = reg2hex[load_reg]; } static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp, @@ -1642,6 +1652,7 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, if (load_probe.prg != -1) { probe.prg = jit->prg; probe.arena_reg = load_probe.arena_reg; + probe.reg = load_probe.reg; } loop_start = jit->prg; /* 0: {csy|csg} %w0,%src,off(%arena) */ @@ -1783,8 +1794,8 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, insn->imm == BPF_FUNC_get_smp_processor_id) { const u32 *cpu_nr = &get_lowcore()->cpu_nr; - /* ly %b0, cpu_nr */ - EMIT6_DISP_LH(0xe3000000, 0x0058, BPF_REG_0, REG_0, REG_0, + /* llgf %b0, cpu_nr */ + EMIT6_DISP_LH(0xe3000000, 0x0016, BPF_REG_0, REG_0, REG_0, (unsigned long)cpu_nr); break; } diff --git a/arch/x86/entry/entry_64_fred.S b/arch/x86/entry/entry_64_fred.S index 0d2768ab836c46..b98f8945dfff46 100644 --- a/arch/x86/entry/entry_64_fred.S +++ b/arch/x86/entry/entry_64_fred.S @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -19,6 +20,7 @@ UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR PUSH_AND_CLEAR_REGS + ENCODE_FRAME_POINTER movq %rsp, %rdi /* %rdi -> pt_regs */ .endm diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index dbc00b6dd69ee2..222dfab9225fe1 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -265,6 +265,29 @@ static void amd_uncore_del(struct perf_event *event, int flags) hwc->idx = -1; } +static bool amd_uncore_group_valid(struct perf_event *event) +{ + struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event); + struct perf_event *leader = event->group_leader; + struct perf_event *sibling; + int counters = 0; + + if (leader->pmu == event->pmu) + counters++; + + for_each_sibling_event(sibling, leader) { + if (sibling->pmu == event->pmu && + sibling->state > PERF_EVENT_STATE_OFF) + counters++; + } + + /* + * When pmu->event_init() is called, the event is yet to be linked to + * its leader's sibling list, so it is counted separately + */ + return (counters + 1) <= pmu->num_counters; +} + static int amd_uncore_event_init(struct perf_event *event) { struct amd_uncore_pmu *pmu; @@ -282,6 +305,14 @@ static int amd_uncore_event_init(struct perf_event *event) if (!ctx) return -ENODEV; + /* + * Ensure that all events in a group can be scheduled together so that + * a failure can be reported at perf_event_open() time rather than + * silently at pmu->add() time when no free counter is found + */ + if (event->group_leader != event && !amd_uncore_group_valid(event)) + return -EINVAL; + /* * NB and Last level cache counters (MSRs) are shared across all cores * that share the same NB / Last level cache. On family 16h and below, diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index d1af33d96d0a31..af0b67ffb43d8a 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -2539,7 +2539,8 @@ static int x86_pmu_event_init(struct perf_event *event) } if (READ_ONCE(x86_pmu.attr_rdpmc) && - !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS)) + !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS) && + !(event->hw.config & ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE)) event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT; return err; diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 465c414f145da0..5116b15438a211 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -3533,7 +3533,7 @@ static void intel_pmu_update_rdpmc_user_disable(struct perf_event *event) */ if (x86_pmu.attr_rdpmc == X86_USER_RDPMC_ALWAYS_ENABLE || (x86_pmu.attr_rdpmc == X86_USER_RDPMC_CONDITIONAL_ENABLE && - event->ctx->task)) + (event->attach_state & PERF_ATTACH_TASK))) event->hw.config &= ~ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE; else event->hw.config |= ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE; @@ -3547,8 +3547,6 @@ static void intel_pmu_enable_event(struct perf_event *event) struct hw_perf_event *hwc = &event->hw; int idx = hwc->idx; - intel_pmu_update_rdpmc_user_disable(event); - if (unlikely(event->attr.precise_ip)) static_call(x86_pmu_pebs_enable)(event); @@ -5147,6 +5145,8 @@ static int intel_pmu_hw_config(struct perf_event *event) leader->hw.flags |= PERF_X86_EVENT_ACR; } + intel_pmu_update_rdpmc_user_disable(event); + if ((event->attr.type == PERF_TYPE_HARDWARE) || (event->attr.type == PERF_TYPE_HW_CACHE)) return 0; diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c index b5726b50e77dbe..5754cd40556281 100644 --- a/arch/x86/events/intel/pt.c +++ b/arch/x86/events/intel/pt.c @@ -502,6 +502,29 @@ static u64 pt_config_filters(struct perf_event *event) return rtit_ctl; } +static void pt_config_enable(struct perf_event *event) +{ + struct pt *pt = this_cpu_ptr(&pt_ctx); + + /* + * Allow resume before starting so as not to overwrite a value set by a + * PMI. + */ + barrier(); + WRITE_ONCE(pt->resume_allowed, 1); + /* Configuration is complete, it is now OK to handle an NMI */ + barrier(); + WRITE_ONCE(pt->handle_nmi, 1); + barrier(); + pt_config_start(event); + barrier(); + /* + * Allow pause after starting so its pt_config_stop() doesn't race with + * pt_config_start(). + */ + WRITE_ONCE(pt->pause_allowed, 1); +} + static void pt_config(struct perf_event *event) { struct pt *pt = this_cpu_ptr(&pt_ctx); @@ -541,23 +564,7 @@ static void pt_config(struct perf_event *event) event->hw.aux_config = reg; - /* - * Allow resume before starting so as not to overwrite a value set by a - * PMI. - */ - barrier(); - WRITE_ONCE(pt->resume_allowed, 1); - /* Configuration is complete, it is now OK to handle an NMI */ - barrier(); - WRITE_ONCE(pt->handle_nmi, 1); - barrier(); - pt_config_start(event); - barrier(); - /* - * Allow pause after starting so its pt_config_stop() doesn't race with - * pt_config_start(). - */ - WRITE_ONCE(pt->pause_allowed, 1); + pt_config_enable(event); } static void pt_config_stop(struct perf_event *event) @@ -1533,12 +1540,14 @@ void intel_pt_interrupt(void) perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0)); - if (!event->hw.state) { + event->hw.state |= PERF_HES_UPTODATE; + + if (!(event->hw.state & PERF_HES_STOPPED)) { int ret; buf = perf_aux_output_begin(&pt->handle, event); if (!buf) { - event->hw.state = PERF_HES_STOPPED; + event->hw.state |= PERF_HES_STOPPED; WRITE_ONCE(pt->resume_allowed, 0); return; } @@ -1554,6 +1563,8 @@ void intel_pt_interrupt(void) pt_config_buffer(buf); pt_config_start(event); + + event->hw.state &= ~PERF_HES_UPTODATE; } } @@ -1622,6 +1633,18 @@ static void pt_event_start(struct perf_event *event, int mode) return; } + /* + * Re-start subsequent to a call to pt_event_stop() without the + * PERF_EF_UPDATE flag. Absence of PERF_HES_UPTODATE indicates that + * perf_aux_output_begin() has already been called. This path can + * come about only in snapshot/overwrite mode - see pt_event_stop(). + */ + if (!(hwc->state & PERF_HES_UPTODATE)) { + hwc->state &= ~PERF_HES_STOPPED; + pt_config_enable(event); + return; + } + buf = perf_aux_output_begin(&pt->handle, event); if (!buf) goto fail_stop; @@ -1632,7 +1655,7 @@ static void pt_event_start(struct perf_event *event, int mode) goto fail_end_stop; } - hwc->state = 0; + hwc->state &= ~(PERF_HES_STOPPED | PERF_HES_UPTODATE); pt_config_buffer(buf); pt_config(event); @@ -1642,12 +1665,13 @@ static void pt_event_start(struct perf_event *event, int mode) fail_end_stop: perf_aux_output_end(&pt->handle, 0); fail_stop: - hwc->state = PERF_HES_STOPPED; + hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; } static void pt_event_stop(struct perf_event *event, int mode) { struct pt *pt = this_cpu_ptr(&pt_ctx); + struct pt_buffer *buf; if (mode & PERF_EF_PAUSE) { if (READ_ONCE(pt->pause_allowed)) @@ -1673,17 +1697,24 @@ static void pt_event_stop(struct perf_event *event, int mode) pt_config_stop(event); - if (event->hw.state == PERF_HES_STOPPED) - return; - - event->hw.state = PERF_HES_STOPPED; + event->hw.state |= PERF_HES_STOPPED; - if (mode & PERF_EF_UPDATE) { - struct pt_buffer *buf = perf_get_aux(&pt->handle); + if (event->hw.state & PERF_HES_UPTODATE) + return; - if (!buf) - return; + buf = perf_get_aux(&pt->handle); + if (!buf) + return; + /* + * When not in snapshot/overwrite mode, there is a possibility that the + * buffer has run out of space. The accounting for that is handled by + * the update, so always update in that case. Snapshot/overwrite mode is + * treated differently to allow for pt_event_snapshot_aux() which can + * still get called if the AUX-sampling event is not stopped until after + * PT is stopped. + */ + if ((mode & PERF_EF_UPDATE) || !buf->snapshot) { if (WARN_ON_ONCE(pt->handle.event != event)) return; @@ -1698,6 +1729,7 @@ static void pt_event_stop(struct perf_event *event, int mode) local_xchg(&buf->data_size, buf->nr_pages << PAGE_SHIFT); perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0)); + event->hw.state |= PERF_HES_UPTODATE; } } @@ -1768,13 +1800,15 @@ static int pt_event_add(struct perf_event *event, int mode) if (pt->handle.event) goto fail; + event->hw.state |= PERF_HES_UPTODATE; + if (mode & PERF_EF_START) { pt_event_start(event, 0); ret = -EINVAL; - if (hwc->state == PERF_HES_STOPPED) + if (hwc->state & PERF_HES_STOPPED) goto fail; } else { - hwc->state = PERF_HES_STOPPED; + hwc->state |= PERF_HES_STOPPED; } ret = 0; diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index 7857959c6e823c..b9ac2f7d31caa0 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -757,7 +757,7 @@ static int uncore_pmu_event_init(struct perf_event *event) pmu = uncore_event_to_pmu(event); /* no device found for this pmu */ - if (!pmu->registered) + if (!uncore_pmu_available(pmu)) return -ENOENT; /* Sampling not supported yet */ @@ -953,16 +953,18 @@ static int uncore_pmu_register(struct intel_uncore_pmu *pmu) ret = perf_pmu_register(&pmu->pmu, pmu->name, -1); if (!ret) - pmu->registered = true; + uncore_pmu_set_registered(pmu); return ret; } static void uncore_pmu_unregister(struct intel_uncore_pmu *pmu) { - if (!pmu->registered) + if (!uncore_pmu_registered(pmu)) return; perf_pmu_unregister(&pmu->pmu); - pmu->registered = false; + + /* Keep PMU_BROKEN_BIT sticky. */ + uncore_pmu_clear_registered(pmu); } static void uncore_free_boxes(struct intel_uncore_pmu *pmu) @@ -1148,6 +1150,44 @@ uncore_pci_find_dev_pmu(struct pci_dev *pdev, const struct pci_device_id *ids) return pmu; } +static int uncore_box_setup(struct intel_uncore_pmu *pmu, + struct intel_uncore_box *box) +{ + int ret; + + if (uncore_pmu_broken(pmu)) + return -ENODEV; + + ret = uncore_box_init(box); + if (ret) + goto err; + + /* First active box registers the pmu. */ + if (atomic_inc_return(&pmu->activeboxes) > 1) + return 0; + + ret = uncore_pmu_register(pmu); + if (ret) { + atomic_dec(&pmu->activeboxes); + goto err; + } + + return 0; +err: + /* + * If any box fails, mark the per-package PMU as broken regardless of + * whether it was registered or not. + * + * Don't decrement refcnt to avoid other in-die CPUs from trying to set + * up the PMU box again. + * + * Don't kfree box; MSR and MMIO boxes are freed at module exit only. + */ + uncore_pmu_set_broken(pmu); + uncore_box_exit(box); + return ret; +} + /* * Register the PMU for a PCI device * @pdev: The PCI device. @@ -1167,26 +1207,22 @@ static int uncore_pci_pmu_register(struct pci_dev *pdev, return -EINVAL; box = uncore_alloc_box(type, NUMA_NO_NODE); - if (!box) + if (!box) { + uncore_pmu_set_broken(pmu); return -ENOMEM; + } atomic_inc(&box->refcnt); box->dieid = die; box->pci_dev = pdev; box->pmu = pmu; - uncore_box_init(box); - pmu->boxes[die] = box; - if (atomic_inc_return(&pmu->activeboxes) > 1) - return 0; - - /* First active box registers the pmu */ - ret = uncore_pmu_register(pmu); - if (ret) { - pmu->boxes[die] = NULL; - uncore_box_exit(box); + ret = uncore_box_setup(pmu, box); + if (!ret) + pmu->boxes[die] = box; + else kfree(box); - } + return ret; } @@ -1248,11 +1284,16 @@ static void uncore_pci_pmu_unregister(struct intel_uncore_pmu *pmu, int die) { struct intel_uncore_box *box = pmu->boxes[die]; + if (!box) + return; + pmu->boxes[die] = NULL; if (atomic_dec_return(&pmu->activeboxes) == 0) uncore_pmu_unregister(pmu); - uncore_box_exit(box); - kfree(box); + if (atomic_dec_return(&box->refcnt) == 0) { + uncore_box_exit(box); + kfree(box); + } } static void uncore_pci_remove(struct pci_dev *pdev) @@ -1272,7 +1313,6 @@ static void uncore_pci_remove(struct pci_dev *pdev) break; } } - WARN_ON_ONCE(i >= UNCORE_EXTRA_PCI_DEV_MAX); return; } @@ -1486,7 +1526,8 @@ static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu, if (old_cpu < 0) { WARN_ON_ONCE(box->cpu != -1); - if (uncore_die_has_box(type, die, pmu->pmu_idx)) { + if (uncore_die_has_box(type, die, pmu->pmu_idx) && + !uncore_pmu_broken(pmu)) { box->cpu = new_cpu; cpumask_set_cpu(new_cpu, &pmu->cpu_mask); } @@ -1494,12 +1535,14 @@ static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu, } WARN_ON_ONCE(box->cpu != -1 && box->cpu != old_cpu); - box->cpu = -1; cpumask_clear_cpu(old_cpu, &pmu->cpu_mask); - if (new_cpu < 0) + if (new_cpu < 0) { + box->cpu = -1; continue; + } - if (!uncore_die_has_box(type, die, pmu->pmu_idx)) + /* An inactive box doesn't need migration. */ + if (box->cpu == -1) continue; uncore_pmu_cancel_hrtimer(box); perf_pmu_migrate_context(&pmu->pmu, old_cpu, new_cpu); @@ -1515,7 +1558,7 @@ static void uncore_change_context(struct intel_uncore_type **uncores, uncore_change_type_ctx(*uncores, old_cpu, new_cpu); } -static void uncore_box_unref(struct intel_uncore_type **types, int id) +static void uncore_box_unref(struct intel_uncore_type **types, int die) { struct intel_uncore_type *type; struct intel_uncore_pmu *pmu; @@ -1526,7 +1569,7 @@ static void uncore_box_unref(struct intel_uncore_type **types, int id) type = *types; pmu = type->pmus; for (i = 0; i < type->num_boxes; i++, pmu++) { - box = pmu->boxes[id]; + box = pmu->boxes[die]; if (box && box->cpu >= 0 && atomic_dec_return(&box->refcnt) == 0) uncore_box_exit(box); } @@ -1537,9 +1580,15 @@ static int uncore_event_cpu_offline(unsigned int cpu) { int die, target; + /* Clear the references */ + die = topology_logical_die_id(cpu); + uncore_box_unref(uncore_msr_uncores, die); + uncore_box_unref(uncore_mmio_uncores, die); + /* Check if exiting cpu is used for collecting uncore events */ if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask)) - goto unref; + return 0; + /* Find a new cpu to collect uncore events */ target = cpumask_any_but(topology_die_cpumask(cpu), cpu); @@ -1552,16 +1601,10 @@ static int uncore_event_cpu_offline(unsigned int cpu) uncore_change_context(uncore_msr_uncores, cpu, target); uncore_change_context(uncore_mmio_uncores, cpu, target); uncore_change_context(uncore_pci_uncores, cpu, target); - -unref: - /* Clear the references */ - die = topology_logical_die_id(cpu); - uncore_box_unref(uncore_msr_uncores, die); - uncore_box_unref(uncore_mmio_uncores, die); return 0; } -static int allocate_boxes(struct intel_uncore_type **types, +static void allocate_boxes(struct intel_uncore_type **types, unsigned int die, unsigned int cpu) { struct intel_uncore_box *box, *tmp; @@ -1575,11 +1618,13 @@ static int allocate_boxes(struct intel_uncore_type **types, type = *types; pmu = type->pmus; for (i = 0; i < type->num_boxes; i++, pmu++) { - if (pmu->boxes[die]) + if (pmu->boxes[die] || uncore_pmu_broken(pmu)) continue; box = uncore_alloc_box(type, cpu_to_node(cpu)); - if (!box) + if (!box) { + uncore_pmu_set_broken(pmu); goto cleanup; + } box->pmu = pmu; box->dieid = die; list_add(&box->active_list, &allocated); @@ -1590,33 +1635,28 @@ static int allocate_boxes(struct intel_uncore_type **types, list_del_init(&box->active_list); box->pmu->boxes[die] = box; } - return 0; + return; cleanup: list_for_each_entry_safe(box, tmp, &allocated, active_list) { list_del_init(&box->active_list); kfree(box); } - return -ENOMEM; } static int uncore_box_ref(struct intel_uncore_type **types, - int id, unsigned int cpu) + int die, unsigned int cpu) { struct intel_uncore_type *type; struct intel_uncore_pmu *pmu; struct intel_uncore_box *box; - int i, ret; - - ret = allocate_boxes(types, id, cpu); - if (ret) - return ret; + int i; for (; *types; types++) { type = *types; pmu = type->pmus; for (i = 0; i < type->num_boxes; i++, pmu++) { - box = pmu->boxes[id]; + box = pmu->boxes[die]; if (box && box->cpu >= 0 && atomic_inc_return(&box->refcnt) == 1) uncore_box_init(box); } @@ -1626,29 +1666,26 @@ static int uncore_box_ref(struct intel_uncore_type **types, static int uncore_event_cpu_online(unsigned int cpu) { - int die, target, msr_ret, mmio_ret; + int die, target; die = topology_logical_die_id(cpu); - msr_ret = uncore_box_ref(uncore_msr_uncores, die, cpu); - mmio_ret = uncore_box_ref(uncore_mmio_uncores, die, cpu); - if (msr_ret && mmio_ret) - return -ENOMEM; + allocate_boxes(uncore_msr_uncores, die, cpu); + allocate_boxes(uncore_mmio_uncores, die, cpu); /* * Check if there is an online cpu in the package * which collects uncore events already. */ target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu)); - if (target < nr_cpu_ids) - return 0; - - cpumask_set_cpu(cpu, &uncore_cpu_mask); - - if (!msr_ret) + if (target >= nr_cpu_ids) { + cpumask_set_cpu(cpu, &uncore_cpu_mask); uncore_change_context(uncore_msr_uncores, -1, cpu); - if (!mmio_ret) uncore_change_context(uncore_mmio_uncores, -1, cpu); - uncore_change_context(uncore_pci_uncores, -1, cpu); + uncore_change_context(uncore_pci_uncores, -1, cpu); + } + + uncore_box_ref(uncore_msr_uncores, die, cpu); + uncore_box_ref(uncore_mmio_uncores, die, cpu); return 0; } diff --git a/arch/x86/events/intel/uncore.h b/arch/x86/events/intel/uncore.h index c2e5ccb1d72c4b..0adb477d970864 100644 --- a/arch/x86/events/intel/uncore.h +++ b/arch/x86/events/intel/uncore.h @@ -129,7 +129,7 @@ struct intel_uncore_type { #define events_group attr_groups[2] struct intel_uncore_ops { - void (*init_box)(struct intel_uncore_box *); + int (*init_box)(struct intel_uncore_box *); void (*exit_box)(struct intel_uncore_box *); void (*disable_box)(struct intel_uncore_box *); void (*enable_box)(struct intel_uncore_box *); @@ -146,13 +146,24 @@ struct intel_uncore_pmu { struct pmu pmu; char name[UNCORE_PMU_NAME_LEN]; int pmu_idx; - bool registered; + unsigned long flags; atomic_t activeboxes; cpumask_t cpu_mask; struct intel_uncore_type *type; struct intel_uncore_box **boxes; }; +#define PMU_REGISTERED_BIT 0 +#define PMU_BROKEN_BIT 1 + +#define uncore_pmu_registered(pmu) test_bit(PMU_REGISTERED_BIT, &(pmu)->flags) +#define uncore_pmu_broken(pmu) test_bit(PMU_BROKEN_BIT, &(pmu)->flags) +#define uncore_pmu_available(pmu) (uncore_pmu_registered(pmu) && \ + !uncore_pmu_broken(pmu)) +#define uncore_pmu_set_registered(pmu) set_bit(PMU_REGISTERED_BIT, &(pmu)->flags) +#define uncore_pmu_set_broken(pmu) set_bit(PMU_BROKEN_BIT, &(pmu)->flags) +#define uncore_pmu_clear_registered(pmu) clear_bit(PMU_REGISTERED_BIT, &(pmu)->flags) + struct intel_uncore_extra_reg { raw_spinlock_t lock; u64 config, config1, config2; @@ -185,7 +196,7 @@ struct intel_uncore_box { #define CFL_UNC_CBO_7_PERFEVTSEL0 0xf70 #define CFL_UNC_CBO_7_PER_CTR0 0xf76 -#define UNCORE_BOX_FLAG_INITIATED 0 +#define UNCORE_BOX_FLAG_INITIALIZED 0 /* event config registers are 8-byte apart */ #define UNCORE_BOX_FLAG_CTL_OFFS8 1 /* CFL 8th CBOX has different MSR space */ @@ -557,17 +568,23 @@ static inline u64 uncore_read_counter(struct intel_uncore_box *box, return box->pmu->type->ops->read_counter(box, event); } -static inline void uncore_box_init(struct intel_uncore_box *box) +static inline int uncore_box_init(struct intel_uncore_box *box) { - if (!test_and_set_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) { - if (box->pmu->type->ops->init_box) - box->pmu->type->ops->init_box(box); + int ret = 0; + + if (!test_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags) && + box->pmu->type->ops->init_box) { + ret = box->pmu->type->ops->init_box(box); + if (!ret) + __set_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags); } + + return ret; } static inline void uncore_box_exit(struct intel_uncore_box *box) { - if (test_and_clear_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) { + if (test_and_clear_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags)) { if (box->pmu->type->ops->exit_box) box->pmu->type->ops->exit_box(box); } diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index e5077622225686..0a22edf4d509ad 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -489,14 +489,15 @@ static u64 intel_generic_uncore_box_ctl(struct intel_uncore_box *box) return unit->addr; } -void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box) +int intel_generic_uncore_msr_init_box(struct intel_uncore_box *box) { u64 box_ctl = intel_generic_uncore_box_ctl(box); if (!box_ctl) - return; + return -ENODEV; wrmsrq(box_ctl, GENERIC_PMON_BOX_CTL_INT); + return 0; } void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box) @@ -578,15 +579,16 @@ static inline int intel_pci_uncore_box_ctl(struct intel_uncore_box *box) return UNCORE_DISCOVERY_PCI_BOX_CTRL(intel_generic_uncore_box_ctl(box)); } -void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box) +int intel_generic_uncore_pci_init_box(struct intel_uncore_box *box) { int box_ctl = intel_pci_uncore_box_ctl(box); if (!box_ctl) - return; + return -ENODEV; __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - pci_write_config_dword(box->pci_dev, box_ctl, GENERIC_PMON_BOX_CTL_INT); + return pci_write_config_dword(box->pci_dev, box_ctl, + GENERIC_PMON_BOX_CTL_INT); } void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box) @@ -648,7 +650,7 @@ static struct intel_uncore_ops generic_uncore_pci_ops = { #define UNCORE_GENERIC_MMIO_SIZE 0x4000 -void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) +int intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) { static struct intel_uncore_discovery_unit *unit; struct intel_uncore_type *type = box->pmu->type; @@ -658,13 +660,13 @@ void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) if (!unit) { pr_warn("Uncore type %d id %d: Cannot find box control address.\n", type->type_id, box->pmu->pmu_idx); - return; + return -ENODEV; } if (!unit->addr) { pr_warn("Uncore type %d box %d: Invalid box control address.\n", type->type_id, unit->id); - return; + return -ENODEV; } addr = unit->addr; @@ -672,10 +674,11 @@ void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) if (!box->io_addr) { pr_warn("Uncore type %d box %d: ioremap error for 0x%llx.\n", type->type_id, unit->id, (unsigned long long)addr); - return; + return -ENOMEM; } writel(GENERIC_PMON_BOX_CTL_INT, box->io_addr); + return 0; } void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box) diff --git a/arch/x86/events/intel/uncore_discovery.h b/arch/x86/events/intel/uncore_discovery.h index e1330342b92ee7..142e1b56cfc2e9 100644 --- a/arch/x86/events/intel/uncore_discovery.h +++ b/arch/x86/events/intel/uncore_discovery.h @@ -148,11 +148,11 @@ void intel_uncore_generic_uncore_cpu_init(void); int intel_uncore_generic_uncore_pci_init(void); void intel_uncore_generic_uncore_mmio_init(void); -void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box); +int intel_generic_uncore_msr_init_box(struct intel_uncore_box *box); void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box); void intel_generic_uncore_msr_enable_box(struct intel_uncore_box *box); -void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box); +int intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box); void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box); void intel_generic_uncore_mmio_enable_box(struct intel_uncore_box *box); void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box, @@ -160,7 +160,7 @@ void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box, void intel_generic_uncore_mmio_enable_event(struct intel_uncore_box *box, struct perf_event *event); -void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box); +int intel_generic_uncore_pci_init_box(struct intel_uncore_box *box); void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box); void intel_generic_uncore_pci_enable_box(struct intel_uncore_box *box); void intel_generic_uncore_pci_disable_event(struct intel_uncore_box *box, diff --git a/arch/x86/events/intel/uncore_nhmex.c b/arch/x86/events/intel/uncore_nhmex.c index 8962e7cb21e3ec..7a6855281102f0 100644 --- a/arch/x86/events/intel/uncore_nhmex.c +++ b/arch/x86/events/intel/uncore_nhmex.c @@ -199,9 +199,10 @@ DEFINE_UNCORE_FORMAT_ATTR(counter, counter, "config:6-7"); DEFINE_UNCORE_FORMAT_ATTR(match, match, "config1:0-63"); DEFINE_UNCORE_FORMAT_ATTR(mask, mask, "config2:0-63"); -static void nhmex_uncore_msr_init_box(struct intel_uncore_box *box) +static int nhmex_uncore_msr_init_box(struct intel_uncore_box *box) { wrmsrq(NHMEX_U_MSR_PMON_GLOBAL_CTL, NHMEX_U_PMON_GLOBAL_EN_ALL); + return 0; } static void nhmex_uncore_msr_exit_box(struct intel_uncore_box *box) diff --git a/arch/x86/events/intel/uncore_snb.c b/arch/x86/events/intel/uncore_snb.c index edddd4f9ab5fcb..055131c508ffae 100644 --- a/arch/x86/events/intel/uncore_snb.c +++ b/arch/x86/events/intel/uncore_snb.c @@ -295,12 +295,14 @@ static void snb_uncore_msr_disable_event(struct intel_uncore_box *box, struct pe wrmsrq(event->hw.config_base, 0); } -static void snb_uncore_msr_init_box(struct intel_uncore_box *box) +static int snb_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) { wrmsrq(SNB_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN | SNB_UNC_GLOBAL_CTL_CORE_ALL); } + + return 0; } static void snb_uncore_msr_enable_box(struct intel_uncore_box *box) @@ -394,7 +396,7 @@ void snb_uncore_cpu_init(void) snb_uncore_cbox.num_boxes = topology_num_cores_per_package(); } -static void skl_uncore_msr_init_box(struct intel_uncore_box *box) +static int skl_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) { wrmsrq(SKL_UNC_PERF_GLOBAL_CTL, @@ -404,6 +406,8 @@ static void skl_uncore_msr_init_box(struct intel_uncore_box *box) /* The 8th CBOX has different MSR space */ if (box->pmu->pmu_idx == 7) __set_bit(UNCORE_BOX_FLAG_CFL8_CBOX_MSR_OFFS, &box->flags); + + return 0; } static void skl_uncore_msr_enable_box(struct intel_uncore_box *box) @@ -547,10 +551,12 @@ static struct intel_uncore_type *tgl_msr_uncores[] = { NULL, }; -static void rkl_uncore_msr_init_box(struct intel_uncore_box *box) +static int rkl_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) wrmsrq(SKL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); + + return 0; } void tgl_uncore_cpu_init(void) @@ -707,9 +713,10 @@ static struct intel_uncore_type mtl_uncore_hac_cbox = { .format_group = &adl_uncore_format_group, }; -static void mtl_uncore_msr_init_box(struct intel_uncore_box *box) +static int mtl_uncore_msr_init_box(struct intel_uncore_box *box) { wrmsrq(uncore_msr_box_ctl(box), SNB_UNC_GLOBAL_CTL_EN); + return 0; } static struct intel_uncore_ops mtl_uncore_msr_ops = { @@ -773,10 +780,12 @@ static struct intel_uncore_type *lnl_msr_uncores[] = { #define LNL_UNC_MSR_GLOBAL_CTL 0x240e -static void lnl_uncore_msr_init_box(struct intel_uncore_box *box) +static int lnl_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) wrmsrq(LNL_UNC_MSR_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); + + return 0; } static struct intel_uncore_ops lnl_uncore_msr_ops = { @@ -874,7 +883,7 @@ static const struct attribute_group snb_uncore_imc_format_group = { .attrs = snb_uncore_imc_formats_attr, }; -static void snb_uncore_imc_init_box(struct intel_uncore_box *box) +static int snb_uncore_imc_init_box(struct intel_uncore_box *box) { struct intel_uncore_type *type = box->pmu->type; struct pci_dev *pdev = box->pci_dev; @@ -893,10 +902,13 @@ static void snb_uncore_imc_init_box(struct intel_uncore_box *box) addr &= ~(PAGE_SIZE - 1); box->io_addr = ioremap(addr, type->mmio_map_size); - if (!box->io_addr) + if (!box->io_addr) { pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name); + return -ENOMEM; + } box->hrtimer_duration = UNCORE_SNB_IMC_HRTIMER_INTERVAL; + return 0; } static void snb_uncore_imc_enable_box(struct intel_uncore_box *box) @@ -928,7 +940,7 @@ static int snb_uncore_imc_event_init(struct perf_event *event) pmu = uncore_event_to_pmu(event); /* no device found for this pmu */ - if (!pmu->registered) + if (!uncore_pmu_available(pmu)) return -ENOENT; /* Sampling not supported yet */ @@ -1532,7 +1544,7 @@ static struct pci_dev *tgl_uncore_get_mc_dev(void) #define TGL_UNCORE_MMIO_IMC_MEM_OFFSET 0x10000 #define TGL_UNCORE_PCI_IMC_MAP_SIZE 0xe000 -static void +static int uncore_get_box_mmio_addr(struct intel_uncore_box *box, unsigned int base_offset, int bar_offset, int step) @@ -1541,19 +1553,20 @@ uncore_get_box_mmio_addr(struct intel_uncore_box *box, struct intel_uncore_pmu *pmu = box->pmu; struct intel_uncore_type *type = pmu->type; resource_size_t addr; + int ret = 0; u32 bar; if (!pdev) { pr_warn("perf uncore: Cannot find matched IMC device.\n"); - return; + return -ENODEV; } pci_read_config_dword(pdev, bar_offset, &bar); if (!(bar & BIT(0))) { pr_warn("perf uncore: BAR 0x%x is disabled. Failed to map %s counters.\n", bar_offset, type->name); - pci_dev_put(pdev); - return; + ret = -ENODEV; + goto out; } bar &= ~BIT(0); addr = (resource_size_t)(bar + step * pmu->pmu_idx); @@ -1565,23 +1578,26 @@ uncore_get_box_mmio_addr(struct intel_uncore_box *box, addr += base_offset; box->io_addr = ioremap(addr, type->mmio_map_size); - if (!box->io_addr) + if (!box->io_addr) { + ret = -ENOMEM; pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name); - + } +out: pci_dev_put(pdev); + return ret; } -static void __uncore_imc_init_box(struct intel_uncore_box *box, +static int __uncore_imc_init_box(struct intel_uncore_box *box, unsigned int base_offset) { - uncore_get_box_mmio_addr(box, base_offset, + return uncore_get_box_mmio_addr(box, base_offset, SNB_UNCORE_PCI_IMC_BAR_OFFSET, TGL_UNCORE_MMIO_IMC_MEM_OFFSET); } -static void tgl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +static int tgl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) { - __uncore_imc_init_box(box, 0); + return __uncore_imc_init_box(box, 0); } static struct intel_uncore_ops tgl_uncore_imc_freerunning_ops = { @@ -1648,13 +1664,15 @@ void tgl_uncore_mmio_init(void) #define ADL_UNCORE_IMC_CTL_INT (ADL_UNCORE_IMC_CTL_RST_CTRL | \ ADL_UNCORE_IMC_CTL_RST_CTRS) -static void adl_uncore_imc_init_box(struct intel_uncore_box *box) +static int adl_uncore_imc_init_box(struct intel_uncore_box *box) { - __uncore_imc_init_box(box, ADL_UNCORE_IMC_BASE); + int ret = __uncore_imc_init_box(box, ADL_UNCORE_IMC_BASE); /* The global control in MC1 can control both MCs. */ - if (box->io_addr && (box->pmu->pmu_idx == 1)) + if (!ret && (box->pmu->pmu_idx == 1)) writel(ADL_UNCORE_IMC_CTL_INT, box->io_addr + ADL_UNCORE_IMC_GLOBAL_CTL); + + return ret; } static void adl_uncore_mmio_disable_box(struct intel_uncore_box *box) @@ -1731,9 +1749,9 @@ static struct freerunning_counters adl_uncore_imc_freerunning[] = { [ADL_MMIO_UNCORE_IMC_DATA_WRITE] = { 0xA0, 0x0, 0x0, 1, 64 }, }; -static void adl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +static int adl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) { - __uncore_imc_init_box(box, ADL_UNCORE_IMC_FREERUNNING_BASE); + return __uncore_imc_init_box(box, ADL_UNCORE_IMC_FREERUNNING_BASE); } static struct intel_uncore_ops adl_uncore_imc_freerunning_ops = { @@ -1803,9 +1821,9 @@ static const struct attribute_group lnl_uncore_format_group = { .attrs = lnl_uncore_formats_attr, }; -static void lnl_uncore_hbo_init_box(struct intel_uncore_box *box) +static int lnl_uncore_hbo_init_box(struct intel_uncore_box *box) { - uncore_get_box_mmio_addr(box, LNL_UNCORE_HBO_BASE, + return uncore_get_box_mmio_addr(box, LNL_UNCORE_HBO_BASE, LNL_UNCORE_PCI_SAFBAR_OFFSET, LNL_UNCORE_HBO_OFFSET); } @@ -1829,14 +1847,16 @@ static struct intel_uncore_type lnl_uncore_hbo = { .format_group = &lnl_uncore_format_group, }; -static void lnl_uncore_sncu_init_box(struct intel_uncore_box *box) +static int lnl_uncore_sncu_init_box(struct intel_uncore_box *box) { - uncore_get_box_mmio_addr(box, LNL_UNCORE_SNCU_BASE, + int ret = uncore_get_box_mmio_addr(box, LNL_UNCORE_SNCU_BASE, LNL_UNCORE_PCI_SAFBAR_OFFSET, 0); - if (box->io_addr) + if (!ret) writel(ADL_UNCORE_IMC_CTL_INT, box->io_addr + LNL_UNCORE_GLOBAL_CTL); + + return ret; } static struct intel_uncore_ops lnl_uncore_sncu_ops = { @@ -1887,13 +1907,15 @@ static struct intel_uncore_type ptl_uncore_imc = { .mmio_map_size = 0xf00, }; -static void ptl_uncore_sncu_init_box(struct intel_uncore_box *box) +static int ptl_uncore_sncu_init_box(struct intel_uncore_box *box) { - intel_generic_uncore_mmio_init_box(box); + int ret = intel_generic_uncore_mmio_init_box(box); /* Clear the global freeze bit */ if (box->io_addr) writel(0, box->io_addr + PTL_UNCORE_GLOBAL_CTL_OFFSET); + + return ret; } static struct intel_uncore_ops ptl_uncore_sncu_ops = { diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 334dc384b5b936..a97cd029db366e 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -627,12 +627,12 @@ static u64 snbep_uncore_pci_read_counter(struct intel_uncore_box *box, struct pe return count; } -static void snbep_uncore_pci_init_box(struct intel_uncore_box *box) +static int snbep_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; int box_ctl = uncore_pci_box_ctl(box); - pci_write_config_dword(pdev, box_ctl, SNBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, box_ctl, SNBEP_PMON_BOX_CTL_INT); } static void snbep_uncore_msr_disable_box(struct intel_uncore_box *box) @@ -680,12 +680,14 @@ static void snbep_uncore_msr_disable_event(struct intel_uncore_box *box, wrmsrq(hwc->config_base, hwc->config); } -static void snbep_uncore_msr_init_box(struct intel_uncore_box *box) +static int snbep_uncore_msr_init_box(struct intel_uncore_box *box) { unsigned msr = uncore_msr_box_ctl(box); if (msr) wrmsrq(msr, SNBEP_PMON_BOX_CTL_INT); + + return 0; } static struct attribute *snbep_uncore_formats_attr[] = { @@ -1507,18 +1509,21 @@ int snbep_uncore_pci_init(void) /* end of Sandy Bridge-EP uncore support */ /* IvyTown uncore support */ -static void ivbep_uncore_msr_init_box(struct intel_uncore_box *box) +static int ivbep_uncore_msr_init_box(struct intel_uncore_box *box) { unsigned msr = uncore_msr_box_ctl(box); if (msr) wrmsrq(msr, IVBEP_PMON_BOX_CTL_INT); + + return 0; } -static void ivbep_uncore_pci_init_box(struct intel_uncore_box *box) +static int ivbep_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; - pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL, + IVBEP_PMON_BOX_CTL_INT); } #define IVBEP_UNCORE_MSR_OPS_COMMON_INIT() \ @@ -2784,7 +2789,7 @@ static struct intel_uncore_type hswep_uncore_cbox = { /* * Write SBOX Initialization register bit by bit to avoid spurious #GPs */ -static void hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box) +static int hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box) { unsigned msr = uncore_msr_box_ctl(box); @@ -2798,6 +2803,8 @@ static void hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box) wrmsrq(msr, flags); } } + + return 0; } static struct intel_uncore_ops hswep_uncore_sbox_msr_ops = { @@ -4162,12 +4169,13 @@ static const struct attribute_group skx_upi_uncore_format_group = { .attrs = skx_upi_uncore_formats_attr, }; -static void skx_upi_uncore_pci_init_box(struct intel_uncore_box *box) +static int skx_upi_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - pci_write_config_dword(pdev, SKX_UPI_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, SKX_UPI_PCI_PMON_BOX_CTL, + IVBEP_PMON_BOX_CTL_INT); } static struct intel_uncore_ops skx_upi_uncore_pci_ops = { @@ -4323,12 +4331,13 @@ static struct intel_uncore_type skx_uncore_upi = { .cleanup_mapping = skx_upi_cleanup_mapping, }; -static void skx_m2m_uncore_pci_init_box(struct intel_uncore_box *box) +static int skx_m2m_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - pci_write_config_dword(pdev, SKX_M2M_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, SKX_M2M_PCI_PMON_BOX_CTL, + IVBEP_PMON_BOX_CTL_INT); } static struct intel_uncore_ops skx_m2m_uncore_pci_ops = { @@ -4831,13 +4840,13 @@ void snr_uncore_cpu_init(void) uncore_msr_uncores = snr_msr_uncores; } -static void snr_m2m_uncore_pci_init_box(struct intel_uncore_box *box) +static int snr_m2m_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; int box_ctl = uncore_pci_box_ctl(box); __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - pci_write_config_dword(pdev, box_ctl, IVBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, box_ctl, IVBEP_PMON_BOX_CTL_INT); } static struct intel_uncore_ops snr_m2m_uncore_pci_ops = { @@ -5010,17 +5019,22 @@ static int snr_uncore_mmio_map(struct intel_uncore_box *box, return 0; } -static void __snr_uncore_mmio_init_box(struct intel_uncore_box *box, +static int __snr_uncore_mmio_init_box(struct intel_uncore_box *box, unsigned int box_ctl, int mem_offset, unsigned int device) { - if (!snr_uncore_mmio_map(box, box_ctl, mem_offset, device)) + int ret; + + ret = snr_uncore_mmio_map(box, box_ctl, mem_offset, device); + if (!ret) writel(IVBEP_PMON_BOX_CTL_INT, box->io_addr); + + return ret; } -static void snr_uncore_mmio_init_box(struct intel_uncore_box *box) +static int snr_uncore_mmio_init_box(struct intel_uncore_box *box) { - __snr_uncore_mmio_init_box(box, uncore_mmio_box_ctl(box), + return __snr_uncore_mmio_init_box(box, uncore_mmio_box_ctl(box), SNR_IMC_MMIO_MEM0_OFFSET, SNR_MC_DEVICE_ID); } @@ -5637,14 +5651,14 @@ int icx_uncore_pci_init(void) return 0; } -static void icx_uncore_imc_init_box(struct intel_uncore_box *box) +static int icx_uncore_imc_init_box(struct intel_uncore_box *box) { unsigned int box_ctl = box->pmu->type->box_ctl + box->pmu->type->mmio_offset * (box->pmu->pmu_idx % ICX_NUMBER_IMC_CHN); int mem_offset = (box->pmu->pmu_idx / ICX_NUMBER_IMC_CHN) * ICX_IMC_MEM_STRIDE + SNR_IMC_MMIO_MEM0_OFFSET; - __snr_uncore_mmio_init_box(box, box_ctl, mem_offset, + return __snr_uncore_mmio_init_box(box, box_ctl, mem_offset, SNR_MC_DEVICE_ID); } @@ -5701,12 +5715,12 @@ static struct uncore_event_desc icx_uncore_imc_freerunning_events[] = { { /* end: all zeroes */ }, }; -static void icx_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +static int icx_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) { int mem_offset = box->pmu->pmu_idx * ICX_IMC_MEM_STRIDE + SNR_IMC_MMIO_MEM0_OFFSET; - snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box), + return snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box), mem_offset, SNR_MC_DEVICE_ID); } @@ -6003,10 +6017,10 @@ static struct intel_uncore_type spr_uncore_mdf = { .name = "mdf", }; -static void spr_uncore_mmio_offs8_init_box(struct intel_uncore_box *box) +static int spr_uncore_mmio_offs8_init_box(struct intel_uncore_box *box) { __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - intel_generic_uncore_mmio_init_box(box); + return intel_generic_uncore_mmio_init_box(box); } static struct intel_uncore_ops spr_uncore_mmio_offs8_ops = { @@ -6187,12 +6201,11 @@ static struct uncore_event_desc spr_uncore_imc_freerunning_events[] = { #define SPR_MC_DEVICE_ID 0x3251 -static void spr_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +static int spr_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) { int mem_offset = box->pmu->pmu_idx * ICX_IMC_MEM_STRIDE + SNR_IMC_MMIO_MEM0_OFFSET; - - snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box), - mem_offset, SPR_MC_DEVICE_ID); + return snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box), + mem_offset, SPR_MC_DEVICE_ID); } static struct intel_uncore_ops spr_uncore_imc_freerunning_ops = { @@ -6881,20 +6894,24 @@ static unsigned int dmr_iio_freerunning_box_offsets[] = { 0x0, 0x8000, 0x18000, 0x20000 }; -static void dmr_uncore_freerunning_init_box(struct intel_uncore_box *box) +static int dmr_uncore_freerunning_init_box(struct intel_uncore_box *box) { struct intel_uncore_type *type = box->pmu->type; u64 mmio_base; if (box->pmu->pmu_idx >= type->num_boxes) - return; + return -ENODEV; mmio_base = DMR_IMH1_HIOP_MMIO_BASE; mmio_base += dmr_iio_freerunning_box_offsets[box->pmu->pmu_idx]; box->io_addr = ioremap(mmio_base, type->mmio_map_size); - if (!box->io_addr) + if (!box->io_addr) { pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name); + return -ENOMEM; + } + + return 0; } static struct intel_uncore_ops dmr_uncore_freerunning_ops = { diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 5902a297daa15c..a8afea8d38f0c3 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -1344,7 +1344,7 @@ static inline u64 x86_pmu_get_event_config(struct perf_event *event) static inline bool x86_pmu_has_rdpmc_user_disable(struct pmu *pmu) { return !!(hybrid(pmu, config_mask) & - ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE); + ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE); } extern struct event_constraint emptyconstraint; diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h index a7294656ad908a..c9769a7b6e66c1 100644 --- a/arch/x86/include/asm/linkage.h +++ b/arch/x86/include/asm/linkage.h @@ -103,7 +103,7 @@ .byte 0xb8 ASM_NL \ .long __kcfi_typeid_##name ASM_NL \ CFI_POST_PADDING \ - SYM_FUNC_END(__cfi_##name) + SYM_END(__cfi_##name, SYM_T_FUNC) /* UML needs to be able to override memcpy() and friends for KASAN. */ #ifdef CONFIG_UML diff --git a/arch/x86/include/asm/pkeys.h b/arch/x86/include/asm/pkeys.h index 06ed2cd2592e8e..fcec52132fc7eb 100644 --- a/arch/x86/include/asm/pkeys.h +++ b/arch/x86/include/asm/pkeys.h @@ -88,6 +88,9 @@ int mm_pkey_alloc(struct mm_struct *mm) u16 all_pkeys_mask = ((1U << arch_max_pkey()) - 1); int ret; + if (!arch_pkeys_enabled()) + return -1; + /* * Are we out of pkeys? We must handle this specially * because ffz() behavior is undefined if there are no diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index a3caddd411ec68..4c88bbe82a7efc 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1251,9 +1251,6 @@ static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = { #define VULNBL_INTEL_STEPS(vfm, max_stepping, issues) \ X86_MATCH_VFM_STEPS(vfm, X86_STEP_MIN, max_stepping, issues) -#define VULNBL_INTEL_TYPE(vfm, cpu_type, issues) \ - X86_MATCH_VFM_CPU_TYPE(vfm, INTEL_CPU_TYPE_##cpu_type, issues) - #define VULNBL_AMD(family, blacklist) \ VULNBL(AMD, family, X86_MODEL_ANY, blacklist) @@ -1316,11 +1313,9 @@ static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = { VULNBL_INTEL_STEPS(INTEL_TIGERLAKE, X86_STEP_MAX, GDS | ITS | ITS_NATIVE_ONLY), VULNBL_INTEL_STEPS(INTEL_LAKEFIELD, X86_STEP_MAX, MMIO | MMIO_SBDS | RETBLEED), VULNBL_INTEL_STEPS(INTEL_ROCKETLAKE, X86_STEP_MAX, MMIO | RETBLEED | GDS | ITS | ITS_NATIVE_ONLY), - VULNBL_INTEL_TYPE(INTEL_ALDERLAKE, ATOM, RFDS | VMSCAPE), - VULNBL_INTEL_STEPS(INTEL_ALDERLAKE, X86_STEP_MAX, VMSCAPE), + VULNBL_INTEL_STEPS(INTEL_ALDERLAKE, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_ALDERLAKE_L, X86_STEP_MAX, RFDS | VMSCAPE), - VULNBL_INTEL_TYPE(INTEL_RAPTORLAKE, ATOM, RFDS | VMSCAPE), - VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE, X86_STEP_MAX, VMSCAPE), + VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE_P, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE_S, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_METEORLAKE_L, X86_STEP_MAX, VMSCAPE), @@ -1388,7 +1383,21 @@ static bool __init vulnerable_to_rfds(u64 x86_arch_cap_msr) return true; /* Only consult the blacklist when there is no enumeration: */ - return cpu_matches(cpu_vuln_blacklist, RFDS); + if (!cpu_matches(cpu_vuln_blacklist, RFDS)) + return false; + + /* + * ADL and RPL are affected only if they have Atom CPUs. Hybrids have + * both Core and Atom CPUs. Mark unaffected when Atom CPUs are not + * present. + */ + if ((boot_cpu_data.x86_model == 0x97 || + boot_cpu_data.x86_model == 0xB7) && + boot_cpu_data.topo.intel_type != INTEL_CPU_TYPE_ATOM && + !boot_cpu_has(X86_FEATURE_HYBRID_CPU)) + return false; + + return true; } static bool __init vulnerable_to_its(u64 x86_arch_cap_msr) diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c index 3af979fb41d38d..65a2de82ecd292 100644 --- a/arch/x86/kernel/uprobes.c +++ b/arch/x86/kernel/uprobes.c @@ -276,15 +276,9 @@ static bool is_prefix_bad(struct insn *insn) return false; } -static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool x86_64) +static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn) { - enum insn_mode m = x86_64 ? INSN_MODE_64 : INSN_MODE_32; u32 volatile *good_insns; - int ret; - - ret = insn_decode(insn, auprobe->insn, sizeof(auprobe->insn), m); - if (ret < 0) - return -ENOEXEC; if (is_prefix_bad(insn)) return -ENOTSUPP; @@ -293,7 +287,7 @@ static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool if (insn_masking_exception(insn)) return -ENOTSUPP; - if (x86_64) + if (insn->x86_64) good_insns = good_insns_64; else good_insns = good_insns_32; @@ -631,14 +625,29 @@ static struct vm_special_mapping tramp_mapping = { .pages = tramp_mapping_pages, }; -struct uprobe_trampoline { - struct hlist_node node; - unsigned long vaddr; -}; + +#define LEA_INSN_SIZE 5 +#define OPT_INSN_SIZE (LEA_INSN_SIZE + CALL_INSN_SIZE) +#define REDZONE_SIZE 0x80 + +static const u8 lea_rsp[] = { 0x48, 0x8d, 0x64, 0x24, 0x80 }; + +static bool is_opt_insns(const uprobe_opcode_t *insn) +{ + return !memcmp(insn, lea_rsp, LEA_INSN_SIZE) && + insn[LEA_INSN_SIZE] == CALL_INSN_OPCODE; +} + +static bool is_swbp_opt_insns(uprobe_opcode_t *insn) +{ + return is_swbp_insn(&insn[0]) && + !memcmp(&insn[1], &lea_rsp[1], LEA_INSN_SIZE - 1) && + insn[LEA_INSN_SIZE] == CALL_INSN_OPCODE; +} static bool is_reachable_by_call(unsigned long vtramp, unsigned long vaddr) { - long delta = (long)(vaddr + 5 - vtramp); + long delta = (long)(vaddr + OPT_INSN_SIZE - vtramp); return delta >= INT_MIN && delta <= INT_MAX; } @@ -651,7 +660,7 @@ static unsigned long find_nearest_trampoline(unsigned long vaddr) }; unsigned long low_limit, high_limit; unsigned long low_tramp, high_tramp; - unsigned long call_end = vaddr + 5; + unsigned long call_end = vaddr + OPT_INSN_SIZE; if (check_add_overflow(call_end, INT_MIN, &low_limit)) low_limit = PAGE_SIZE; @@ -682,83 +691,32 @@ static unsigned long find_nearest_trampoline(unsigned long vaddr) return high_tramp; } -static struct uprobe_trampoline *create_uprobe_trampoline(unsigned long vaddr) +static struct vm_area_struct *get_uprobe_trampoline(struct mm_struct *mm, unsigned long vaddr, + bool *new_mapping) { - struct pt_regs *regs = task_pt_regs(current); - struct mm_struct *mm = current->mm; - struct uprobe_trampoline *tramp; + VMA_ITERATOR(vmi, mm, 0); struct vm_area_struct *vma; - if (!user_64bit_mode(regs)) - return NULL; - - vaddr = find_nearest_trampoline(vaddr); - if (IS_ERR_VALUE(vaddr)) - return NULL; - - tramp = kzalloc_obj(*tramp); - if (unlikely(!tramp)) - return NULL; - - tramp->vaddr = vaddr; - vma = _install_special_mapping(mm, tramp->vaddr, PAGE_SIZE, - VM_READ|VM_EXEC|VM_MAYEXEC|VM_MAYREAD|VM_DONTCOPY|VM_IO, - &tramp_mapping); - if (IS_ERR(vma)) { - kfree(tramp); - return NULL; - } - return tramp; -} - -static struct uprobe_trampoline *get_uprobe_trampoline(unsigned long vaddr, bool *new) -{ - struct uprobes_state *state = ¤t->mm->uprobes_state; - struct uprobe_trampoline *tramp = NULL; + *new_mapping = false; if (vaddr > TASK_SIZE || vaddr < PAGE_SIZE) - return NULL; + return ERR_PTR(-EINVAL); - hlist_for_each_entry(tramp, &state->head_tramps, node) { - if (is_reachable_by_call(tramp->vaddr, vaddr)) { - *new = false; - return tramp; - } + for_each_vma(vmi, vma) { + if (!vma_is_special_mapping(vma, &tramp_mapping)) + continue; + if (is_reachable_by_call(vma->vm_start, vaddr)) + return vma; } - tramp = create_uprobe_trampoline(vaddr); - if (!tramp) - return NULL; - - *new = true; - hlist_add_head(&tramp->node, &state->head_tramps); - return tramp; -} - -static void destroy_uprobe_trampoline(struct uprobe_trampoline *tramp) -{ - /* - * We do not unmap and release uprobe trampoline page itself, - * because there's no easy way to make sure none of the threads - * is still inside the trampoline. - */ - hlist_del(&tramp->node); - kfree(tramp); -} - -void arch_uprobe_init_state(struct mm_struct *mm) -{ - INIT_HLIST_HEAD(&mm->uprobes_state.head_tramps); -} - -void arch_uprobe_clear_state(struct mm_struct *mm) -{ - struct uprobes_state *state = &mm->uprobes_state; - struct uprobe_trampoline *tramp; - struct hlist_node *n; + vaddr = find_nearest_trampoline(vaddr); + if (IS_ERR_VALUE(vaddr)) + return ERR_PTR(vaddr); - hlist_for_each_entry_safe(tramp, n, &state->head_tramps, node) - destroy_uprobe_trampoline(tramp); + *new_mapping = true; + return _install_special_mapping(mm, vaddr, PAGE_SIZE, + VM_READ|VM_EXEC|VM_MAYEXEC|VM_MAYREAD|VM_IO, + &tramp_mapping); } static bool __in_uprobe_trampoline(struct mm_struct *mm, unsigned long ip) @@ -810,7 +768,7 @@ SYSCALL_DEFINE0(uprobe) /* Allow execution only from uprobe trampolines. */ if (!in_uprobe_trampoline(regs->ip)) - return -ENXIO; + return -EPROTO; err = copy_from_user(&args, (void __user *)regs->sp, sizeof(args)); if (err) @@ -826,8 +784,8 @@ SYSCALL_DEFINE0(uprobe) regs->ax = args.ax; regs->r11 = args.r11; regs->cx = args.cx; - regs->ip = args.retaddr - 5; - regs->sp += sizeof(args); + regs->ip = args.retaddr - OPT_INSN_SIZE; + regs->sp += sizeof(args) + REDZONE_SIZE; regs->orig_ax = -1; sp = regs->sp; @@ -844,12 +802,12 @@ SYSCALL_DEFINE0(uprobe) */ if (regs->sp != sp) { /* skip the trampoline call */ - if (args.retaddr - 5 == regs->ip) - regs->ip += 5; + if (args.retaddr - OPT_INSN_SIZE == regs->ip) + regs->ip += OPT_INSN_SIZE; return regs->ax; } - regs->sp -= sizeof(args); + regs->sp -= sizeof(args) + REDZONE_SIZE; /* for the case uprobe_consumer has changed ax/r11/cx */ args.ax = regs->ax; @@ -857,7 +815,7 @@ SYSCALL_DEFINE0(uprobe) args.cx = regs->cx; /* keep return address unless we are instructed otherwise */ - if (args.retaddr - 5 != regs->ip) + if (args.retaddr - OPT_INSN_SIZE != regs->ip) args.retaddr = regs->ip; if (shstk_push(args.retaddr) == -EFAULT) @@ -891,7 +849,7 @@ asm ( "pop %rax\n" "pop %r11\n" "pop %rcx\n" - "ret\n" + "ret $" __stringify(REDZONE_SIZE) "\n" "int3\n" ".balign " __stringify(PAGE_SIZE) "\n" ".popsection\n" @@ -909,7 +867,8 @@ late_initcall(arch_uprobes_init); enum { EXPECT_SWBP, - EXPECT_CALL, + EXPECT_OPTIMIZED, + EXPECT_SWBP_OPTIMIZED, }; struct write_opcode_ctx { @@ -917,30 +876,29 @@ struct write_opcode_ctx { int expect; }; -static int is_call_insn(uprobe_opcode_t *insn) -{ - return *insn == CALL_INSN_OPCODE; -} - /* - * Verification callback used by int3_update uprobe_write calls to make sure - * the underlying instruction is as expected - either int3 or call. + * Verification callback used by uprobe_write calls to make sure the underlying + * instruction is in the expected stage of the INT3 update sequence. */ static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode, int nbytes, void *data) { struct write_opcode_ctx *ctx = data; - uprobe_opcode_t old_opcode[5]; + uprobe_opcode_t old_opcode[OPT_INSN_SIZE]; - uprobe_copy_from_page(page, ctx->base, (uprobe_opcode_t *) &old_opcode, 5); + uprobe_copy_from_page(page, ctx->base, old_opcode, OPT_INSN_SIZE); switch (ctx->expect) { case EXPECT_SWBP: if (is_swbp_insn(&old_opcode[0])) return 1; break; - case EXPECT_CALL: - if (is_call_insn(&old_opcode[0])) + case EXPECT_OPTIMIZED: + if (is_opt_insns(&old_opcode[0])) + return 1; + break; + case EXPECT_SWBP_OPTIMIZED: + if (is_swbp_opt_insns(&old_opcode[0])) return 1; break; } @@ -949,48 +907,137 @@ static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t * } /* - * Modify multi-byte instructions by using INT3 breakpoints on SMP. + * Modify the optimized instruction by using INT3 breakpoints on SMP. * We completely avoid using stop_machine() here, and achieve the * synchronization using INT3 breakpoints and SMP cross-calls. * (borrowed comment from smp_text_poke_batch_finish) * - * The way it is done: - * - Add an INT3 trap to the address that will be patched - * - SMP sync all CPUs - * - Update all but the first byte of the patched range - * - SMP sync all CPUs - * - Replace the first byte (INT3) by the first byte of the replacing opcode - * - SMP sync all CPUs + * For optimization (int3_update_optimize): + * 1) Start with the uprobe INT3 trap already installed + * 2) Update everything but the first byte + * 3) Replace the first INT3 by the first byte of the LEA instruction + * + * For unoptimization (int3_update_unoptimize): + * 1) Start with the optimized uprobe lea/call instructions + * 2) Add an INT3 trap to the address that will be patched + * 3) Restore the NOP bytes before the call opcode + * 4) Replace the first INT3 by the first byte of the NOP instruction + * + * Note that unoptimization deliberately keeps the call opcode and displacement + * in bytes 5..9. Those bytes become operands of the restored 10-byte NOP. + * + * Since there is only a single target uprobe-trampoline for the given nop10 + * instruction address, the CALL instruction will not be changed across + * unoptimization/optimization cycles. + * Therefore, any task that is preempted at the CALL instruction is guaranteed + * to observe that CALL and not anything else. */ -static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma, - unsigned long vaddr, char *insn, bool optimize) +static int int3_update_optimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, + unsigned long vaddr, uprobe_opcode_t *insn) { - uprobe_opcode_t int3 = UPROBE_SWBP_INSN; struct write_opcode_ctx ctx = { .base = vaddr, }; int err; /* - * Write int3 trap. + * 1) Initial state after set_swbp() installed the uprobe: + * cc 2e 0f 1f 84 00 00 00 00 00 * - * The swbp_optimize path comes with breakpoint already installed, - * so we can skip this step for optimize == true. + * After a previous unoptimization bytes 5..9 may still contain the + * old call instruction, which remains valid for threads already there. */ - if (!optimize) { - ctx.expect = EXPECT_CALL; - err = uprobe_write(auprobe, vma, vaddr, &int3, 1, verify_insn, - true /* is_register */, false /* do_update_ref_ctr */, - &ctx); - if (err) - return err; - } + smp_text_poke_sync_each_cpu(); + + /* + * 2) Rewrite the LEA tail and call displacement: + * cc [8d 64 24 80 e8 d0 d1 d2 d3] + */ + ctx.expect = EXPECT_SWBP; + err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, + OPT_INSN_SIZE - 1, verify_insn, + true /* is_register */, false /* do_update_ref_ctr */, + &ctx); + if (err) + return err; smp_text_poke_sync_each_cpu(); - /* Write all but the first byte of the patched range. */ + /* + * 3) Publish the first LEA byte: + * [48] 8d 64 24 80 e8 d0 d1 d2 d3 + * + * From offset 0 this is: + * lea -0x80(%rsp), %rsp + * call + */ + ctx.expect = EXPECT_SWBP_OPTIMIZED; + err = uprobe_write(auprobe, vma, vaddr, insn, 1, verify_insn, + true /* is_register */, false /* do_update_ref_ctr */, + &ctx); + if (err) + goto error; + + smp_text_poke_sync_each_cpu(); + return 0; + +error: + /* + * In all intermediate states byte 0 is INT3, so EXPECT_SWBP covers every + * case. Restore NOP bytes 1..4, but keep the valid CALL at bytes 5..9 + * for a thread that had already executed the LEA before a previous + * unoptimization. + */ ctx.expect = EXPECT_SWBP; - err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, 4, verify_insn, + uprobe_write(auprobe, vma, vaddr + 1, auprobe->insn + 1, + LEA_INSN_SIZE - 1, verify_insn, true, false, &ctx); + smp_text_poke_sync_each_cpu(); + return err; +} + +static int int3_update_unoptimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, + unsigned long vaddr, uprobe_opcode_t *insn) +{ + uprobe_opcode_t int3 = UPROBE_SWBP_INSN; + struct write_opcode_ctx ctx = { + .base = vaddr, + .expect = EXPECT_OPTIMIZED, + }; + int err; + + /* + * Note the first two uprobe_write calls use is_register=true, because they + * are intermediate patching states while the probe is still active, so + * we force the exclusive anonymous page for the update. + * Also we use do_update_ref_ctr=false because refctr was already updated by + * the initial int3 install. + * + * The last uprobe_write to nop10 instruction is called with is_register=false + * and do_update_ref_ctr=true to trigger the refctr update and to instruct + * uprobe_write to zap the anonymous page if it now matches the file page. + * + * 1) Initial optimized state: + * 48 8d 64 24 80 e8 d0 d1 d2 d3 + * + * 2) Trap new entries before restoring the NOP bytes: + * [cc] 8d 64 24 80 e8 d0 d1 d2 d3 + */ + err = uprobe_write(auprobe, vma, vaddr, &int3, 1, verify_insn, + true /* is_register */, false /* do_update_ref_ctr */, + &ctx); + if (err) + return err; + + smp_text_poke_sync_each_cpu(); + + /* + * 3) Restore bytes 1..4 of the original NOP while keeping byte 0 trapped + * and byte 5 as CALL: + * cc [2e 0f 1f 84] e8 d0 d1 d2 d3 + */ + ctx.expect = EXPECT_SWBP_OPTIMIZED; + err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, + LEA_INSN_SIZE - 1, verify_insn, true /* is_register */, false /* do_update_ref_ctr */, &ctx); if (err) @@ -999,13 +1046,16 @@ static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma, smp_text_poke_sync_each_cpu(); /* - * Write first byte. + * 4) Publish the first byte of the original NOP: + * [66] 2e 0f 1f 84 e8 d0 d1 d2 d3 * - * The swbp_unoptimize needs to finish uprobe removal together - * with ref_ctr update, using uprobe_write with proper flags. + * From offset 0 this is the restored 10-byte NOP; the CALL opcode and + * displacement are now only NOP operands. Offset 5 still decodes as + * CALL for a thread that was already there. */ + ctx.expect = EXPECT_SWBP; err = uprobe_write(auprobe, vma, vaddr, insn, 1, verify_insn, - optimize /* is_register */, !optimize /* do_update_ref_ctr */, + false /* is_register */, true /* do_update_ref_ctr */, &ctx); if (err) return err; @@ -1017,17 +1067,25 @@ static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma, static int swbp_optimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, unsigned long vaddr, unsigned long tramp) { - u8 call[5]; + u8 insn[OPT_INSN_SIZE], *call = &insn[LEA_INSN_SIZE]; - __text_gen_insn(call, CALL_INSN_OPCODE, (const void *) vaddr, + /* + * We have nop10 instruction (with first byte overwritten to int3), + * changing it to: + * lea -0x80(%rsp), %rsp + * call tramp + */ + memcpy(insn, lea_rsp, LEA_INSN_SIZE); + __text_gen_insn(call, CALL_INSN_OPCODE, + (const void *) (vaddr + LEA_INSN_SIZE), (const void *) tramp, CALL_INSN_SIZE); - return int3_update(auprobe, vma, vaddr, call, true /* optimize */); + return int3_update_optimize(auprobe, vma, vaddr, insn); } static int swbp_unoptimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, unsigned long vaddr) { - return int3_update(auprobe, vma, vaddr, auprobe->insn, false /* optimize */); + return int3_update_unoptimize(auprobe, vma, vaddr, auprobe->insn); } static int copy_from_vaddr(struct mm_struct *mm, unsigned long vaddr, void *dst, int len) @@ -1049,19 +1107,19 @@ static bool __is_optimized(struct mm_struct *mm, uprobe_opcode_t *insn, unsigned struct __packed __arch_relative_insn { u8 op; s32 raddr; - } *call = (struct __arch_relative_insn *) insn; + } *call = (struct __arch_relative_insn *)(insn + LEA_INSN_SIZE); - if (!is_call_insn(insn)) + if (!is_opt_insns(insn)) return false; - return __in_uprobe_trampoline(mm, vaddr + 5 + call->raddr); + return __in_uprobe_trampoline(mm, vaddr + OPT_INSN_SIZE + call->raddr); } static int is_optimized(struct mm_struct *mm, unsigned long vaddr) { - uprobe_opcode_t insn[5]; + uprobe_opcode_t insn[OPT_INSN_SIZE]; int err; - err = copy_from_vaddr(mm, vaddr, &insn, 5); + err = copy_from_vaddr(mm, vaddr, &insn, OPT_INSN_SIZE); if (err) return err; return __is_optimized(mm, (uprobe_opcode_t *)&insn, vaddr); @@ -1111,27 +1169,29 @@ int set_orig_insn(struct arch_uprobe *auprobe, struct vm_area_struct *vma, static int __arch_uprobe_optimize(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) { - struct uprobe_trampoline *tramp; - struct vm_area_struct *vma; - bool new = false; - int err = 0; + struct pt_regs *regs = task_pt_regs(current); + struct vm_area_struct *vma, *tramp; + bool new_mapping; + int ret; + if (!user_64bit_mode(regs)) + return -EINVAL; vma = find_vma(mm, vaddr); if (!vma) return -EINVAL; - tramp = get_uprobe_trampoline(vaddr, &new); - if (!tramp) - return -EINVAL; - err = swbp_optimize(auprobe, vma, vaddr, tramp->vaddr); - if (WARN_ON_ONCE(err) && new) - destroy_uprobe_trampoline(tramp); - return err; + tramp = get_uprobe_trampoline(mm, vaddr, &new_mapping); + if (IS_ERR(tramp)) + return PTR_ERR(tramp); + ret = swbp_optimize(auprobe, vma, vaddr, tramp->vm_start); + if (WARN_ON_ONCE(ret) && new_mapping) + WARN_ON_ONCE(do_munmap(mm, tramp->vm_start, PAGE_SIZE, NULL)); + return ret; } void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr) { struct mm_struct *mm = current->mm; - uprobe_opcode_t insn[5]; + uprobe_opcode_t insn[OPT_INSN_SIZE]; if (!should_optimize(auprobe)) return; @@ -1142,7 +1202,7 @@ void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr) * Check if some other thread already optimized the uprobe for us, * if it's the case just go away silently. */ - if (copy_from_vaddr(mm, vaddr, &insn, 5)) + if (copy_from_vaddr(mm, vaddr, &insn, OPT_INSN_SIZE)) goto unlock; if (!is_swbp_insn((uprobe_opcode_t*) &insn)) goto unlock; @@ -1158,16 +1218,32 @@ unlock: mmap_write_unlock(mm); } +static bool is_optimizable_nop10(struct insn *insn) +{ + static const u8 nop10_prefix[] = { + 0x66, 0x2e, 0x0f, 0x1f, 0x84 + }; + + /* + * Restrict this to the 10-byte NOP form whose last 5 bytes are + * SIB/displacement operands. Unoptimization keeps the call opcode and + * displacement in those bytes, so other NOP encodings are not safe. + */ + return insn->length == OPT_INSN_SIZE && + insn_is_nop(insn) && + !memcmp(insn->kaddr, nop10_prefix, ARRAY_SIZE(nop10_prefix)); +} + static bool can_optimize(struct insn *insn, unsigned long vaddr) { - if (!insn->x86_64 || insn->length != 5) + if (!insn->x86_64) return false; - if (!insn_is_nop(insn)) + if (!is_optimizable_nop10(insn)) return false; /* We can't do cross page atomic writes yet. */ - return PAGE_SIZE - (vaddr & ~PAGE_MASK) >= 5; + return PAGE_SIZE - (vaddr & ~PAGE_MASK) >= OPT_INSN_SIZE; } #else /* 32-bit: */ /* @@ -1549,16 +1625,26 @@ static int push_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn) */ int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long addr) { + enum insn_mode m = is_64bit_mm(mm) ? INSN_MODE_64 : INSN_MODE_32; u8 fix_ip_or_call = UPROBE_FIX_IP; struct insn insn; int ret; - ret = uprobe_init_insn(auprobe, &insn, is_64bit_mm(mm)); - if (ret) - return ret; + ret = insn_decode(&insn, auprobe->insn, sizeof(auprobe->insn), m); + if (ret < 0) + return -ENOEXEC; - if (can_optimize(&insn, addr)) + /* + * No need to check instruction in uprobe_init_insn in case we + * are on top of optimizable nop10. + */ + if (can_optimize(&insn, addr)) { set_bit(ARCH_UPROBE_FLAG_CAN_OPTIMIZE, &auprobe->flags); + } else { + ret = uprobe_init_insn(auprobe, &insn); + if (ret) + return ret; + } ret = branch_setup_xol_ops(auprobe, &insn); if (ret != -ENOSYS) diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 77337c37324b1e..f39c311fd756a7 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -5,8 +5,8 @@ ccflags-$(CONFIG_KVM_WERROR) += -Werror include $(srctree)/virt/kvm/Makefile.kvm -kvm-y += x86.o emulate.o irq.o lapic.o cpuid.o pmu.o mtrr.o \ - debugfs.o mmu/mmu.o mmu/page_track.o mmu/spte.o +kvm-y += x86.o emulate.o irq.o lapic.o cpuid.o pmu.o regs.o \ + mtrr.o debugfs.o mmu/mmu.o mmu/page_track.o mmu/spte.o kvm-$(CONFIG_X86_64) += mmu/tdp_iter.o mmu/tdp_mmu.o kvm-$(CONFIG_KVM_IOAPIC) += i8259.o i8254.o ioapic.o diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c new file mode 100644 index 00000000000000..94c4e4e41868fc --- /dev/null +++ b/arch/x86/kvm/regs.c @@ -0,0 +1,874 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include + +#include "lapic.h" +#include "mmu.h" +#include "regs.h" +#include "x86.h" + +unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu) +{ + /* Can't read the RIP when guest state is protected, just return 0 */ + if (vcpu->arch.guest_state_protected) + return 0; + + if (is_64_bit_mode(vcpu)) + return kvm_rip_read(vcpu); + return (u32)(kvm_get_segment_base(vcpu, VCPU_SREG_CS) + + kvm_rip_read(vcpu)); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_linear_rip); + +bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip) +{ + return kvm_get_linear_rip(vcpu) == linear_rip; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_is_linear_rip); + +unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu) +{ + unsigned long rflags; + + rflags = kvm_x86_call(get_rflags)(vcpu); + if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) + rflags &= ~X86_EFLAGS_TF; + return rflags; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_rflags); + +void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) +{ + if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP && + kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip)) + rflags |= X86_EFLAGS_TF; + kvm_x86_call(set_rflags)(vcpu, rflags); +} + +void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) +{ + __kvm_set_rflags(vcpu, rflags); + kvm_make_request(KVM_REQ_EVENT, vcpu); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_rflags); + +static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + if (vcpu->arch.emulate_regs_need_sync_to_vcpu) { + /* + * We are here if userspace calls get_regs() in the middle of + * instruction emulation. Registers state needs to be copied + * back from emulation context to vcpu. Userspace shouldn't do + * that usually, but some bad designed PV devices (vmware + * backdoor interface) need this to work + */ + emulator_writeback_register_cache(vcpu->arch.emulate_ctxt); + vcpu->arch.emulate_regs_need_sync_to_vcpu = false; + } + regs->rax = kvm_rax_read_raw(vcpu); + regs->rbx = kvm_rbx_read_raw(vcpu); + regs->rcx = kvm_rcx_read_raw(vcpu); + regs->rdx = kvm_rdx_read_raw(vcpu); + regs->rsi = kvm_rsi_read_raw(vcpu); + regs->rdi = kvm_rdi_read_raw(vcpu); + regs->rsp = kvm_rsp_read(vcpu); + regs->rbp = kvm_rbp_read_raw(vcpu); +#ifdef CONFIG_X86_64 + regs->r8 = kvm_r8_read_raw(vcpu); + regs->r9 = kvm_r9_read_raw(vcpu); + regs->r10 = kvm_r10_read_raw(vcpu); + regs->r11 = kvm_r11_read_raw(vcpu); + regs->r12 = kvm_r12_read_raw(vcpu); + regs->r13 = kvm_r13_read_raw(vcpu); + regs->r14 = kvm_r14_read_raw(vcpu); + regs->r15 = kvm_r15_read_raw(vcpu); +#endif + + regs->rip = kvm_rip_read(vcpu); + regs->rflags = kvm_get_rflags(vcpu); +} + +int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + if (vcpu->kvm->arch.has_protected_state && + vcpu->arch.guest_state_protected) + return -EINVAL; + + vcpu_load(vcpu); + __get_regs(vcpu, regs); + vcpu_put(vcpu); + return 0; +} + +static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + vcpu->arch.emulate_regs_need_sync_from_vcpu = true; + vcpu->arch.emulate_regs_need_sync_to_vcpu = false; + + kvm_rax_write_raw(vcpu, regs->rax); + kvm_rbx_write_raw(vcpu, regs->rbx); + kvm_rcx_write_raw(vcpu, regs->rcx); + kvm_rdx_write_raw(vcpu, regs->rdx); + kvm_rsi_write_raw(vcpu, regs->rsi); + kvm_rdi_write_raw(vcpu, regs->rdi); + kvm_rsp_write(vcpu, regs->rsp); + kvm_rbp_write_raw(vcpu, regs->rbp); +#ifdef CONFIG_X86_64 + kvm_r8_write_raw(vcpu, regs->r8); + kvm_r9_write_raw(vcpu, regs->r9); + kvm_r10_write_raw(vcpu, regs->r10); + kvm_r11_write_raw(vcpu, regs->r11); + kvm_r12_write_raw(vcpu, regs->r12); + kvm_r13_write_raw(vcpu, regs->r13); + kvm_r14_write_raw(vcpu, regs->r14); + kvm_r15_write_raw(vcpu, regs->r15); +#endif + + kvm_rip_write(vcpu, regs->rip); + kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED); + + vcpu->arch.exception.pending = false; + vcpu->arch.exception_vmexit.pending = false; + + kvm_make_request(KVM_REQ_EVENT, vcpu); +} + +int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + if (vcpu->kvm->arch.has_protected_state && + vcpu->arch.guest_state_protected) + return -EINVAL; + + vcpu_load(vcpu); + __set_regs(vcpu, regs); + vcpu_put(vcpu); + return 0; +} + +static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2); +} + +/* + * Load the pae pdptrs. Return 1 if they are all valid, 0 otherwise. + */ +int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) +{ + struct kvm_mmu *mmu = vcpu->arch.walk_mmu; + gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; + gpa_t real_gpa; + int i; + int ret; + u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)]; + + /* + * If the MMU is nested, CR3 holds an L2 GPA and needs to be translated + * to an L1 GPA. + */ + real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn), + PFERR_USER_MASK | PFERR_WRITE_MASK | + PFERR_GUEST_PAGE_MASK, NULL, 0); + if (real_gpa == INVALID_GPA) + return 0; + + /* Note the offset, PDPTRs are 32 byte aligned when using PAE paging. */ + ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte, + cr3 & GENMASK(11, 5), sizeof(pdpte)); + if (ret < 0) + return 0; + + for (i = 0; i < ARRAY_SIZE(pdpte); ++i) { + if ((pdpte[i] & PT_PRESENT_MASK) && + (pdpte[i] & pdptr_rsvd_bits(vcpu))) { + return 0; + } + } + + /* + * Marking VCPU_REG_PDPTR dirty doesn't work for !tdp_enabled. + * Shadow page roots need to be reconstructed instead. + */ + if (!tdp_enabled && memcmp(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs))) + kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT); + + memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs)); + kvm_register_mark_dirty(vcpu, VCPU_REG_PDPTR); + kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); + vcpu->arch.pdptrs_from_userspace = false; + + return 1; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(load_pdptrs); + +static bool kvm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) +{ +#ifdef CONFIG_X86_64 + if (cr0 & 0xffffffff00000000UL) + return false; +#endif + + if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) + return false; + + if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) + return false; + + return kvm_x86_call(is_valid_cr0)(vcpu, cr0); +} + +void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0) +{ + /* + * CR0.WP is incorporated into the MMU role, but only for non-nested, + * indirect shadow MMUs. If paging is disabled, no updates are needed + * as there are no permission bits to emulate. If TDP is enabled, the + * MMU's metadata needs to be updated, e.g. so that emulating guest + * translations does the right thing, but there's no need to unload the + * root as CR0.WP doesn't affect SPTEs. + */ + if ((cr0 ^ old_cr0) == X86_CR0_WP) { + if (!(cr0 & X86_CR0_PG)) + return; + + if (tdp_enabled) { + kvm_init_mmu(vcpu); + return; + } + } + + if ((cr0 ^ old_cr0) & X86_CR0_PG) { + /* + * Clearing CR0.PG is defined to flush the TLB from the guest's + * perspective. + */ + if (!(cr0 & X86_CR0_PG)) + kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); + /* + * Check for async #PF completion events when enabling paging, + * as the vCPU may have previously encountered async #PFs (it's + * entirely legal for the guest to toggle paging on/off without + * waiting for the async #PF queue to drain). + */ + else if (kvm_pv_async_pf_enabled(vcpu)) + kvm_make_request(KVM_REQ_APF_READY, vcpu); + } + + if ((cr0 ^ old_cr0) & KVM_MMU_CR0_ROLE_BITS) + kvm_mmu_reset_context(vcpu); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_post_set_cr0); + +int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) +{ + unsigned long old_cr0 = kvm_read_cr0(vcpu); + + if (!kvm_is_valid_cr0(vcpu, cr0)) + return 1; + + cr0 |= X86_CR0_ET; + + /* Write to CR0 reserved bits are ignored, even on Intel. */ + cr0 &= ~CR0_RESERVED_BITS; + +#ifdef CONFIG_X86_64 + if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) && + (cr0 & X86_CR0_PG)) { + int cs_db, cs_l; + + if (!is_pae(vcpu)) + return 1; + kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); + if (cs_l) + return 1; + } +#endif + if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) && + is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) && + !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) + return 1; + + if (!(cr0 & X86_CR0_PG) && + (is_64_bit_mode(vcpu) || kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE))) + return 1; + + if (!(cr0 & X86_CR0_WP) && kvm_is_cr4_bit_set(vcpu, X86_CR4_CET)) + return 1; + + kvm_x86_call(set_cr0)(vcpu, cr0); + + kvm_post_set_cr0(vcpu, old_cr0, cr0); + + return 0; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr0); + +void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw) +{ + (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f)); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_lmsw); + +int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) +{ + bool skip_tlb_flush = false; + unsigned long pcid = 0; +#ifdef CONFIG_X86_64 + if (kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)) { + skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH; + cr3 &= ~X86_CR3_PCID_NOFLUSH; + pcid = cr3 & X86_CR3_PCID_MASK; + } +#endif + + /* PDPTRs are always reloaded for PAE paging. */ + if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu)) + goto handle_tlb_flush; + + /* + * Do not condition the GPA check on long mode, this helper is used to + * stuff CR3, e.g. for RSM emulation, and there is no guarantee that + * the current vCPU mode is accurate. + */ + if (!kvm_vcpu_is_legal_cr3(vcpu, cr3)) + return 1; + + if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3)) + return 1; + + if (cr3 != kvm_read_cr3(vcpu)) + kvm_mmu_new_pgd(vcpu, cr3); + + vcpu->arch.cr3 = cr3; + kvm_register_mark_dirty(vcpu, VCPU_REG_CR3); + /* Do not call post_set_cr3, we do not get here for confidential guests. */ + +handle_tlb_flush: + /* + * A load of CR3 that flushes the TLB flushes only the current PCID, + * even if PCID is disabled, in which case PCID=0 is flushed. It's a + * moot point in the end because _disabling_ PCID will flush all PCIDs, + * and it's impossible to use a non-zero PCID when PCID is disabled, + * i.e. only PCID=0 can be relevant. + */ + if (!skip_tlb_flush) + kvm_invalidate_pcid(vcpu, pcid); + + return 0; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr3); + +static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) +{ + return __kvm_is_valid_cr4(vcpu, cr4) && + kvm_x86_call(is_valid_cr4)(vcpu, cr4); +} + +void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4) +{ + if ((cr4 ^ old_cr4) & KVM_MMU_CR4_ROLE_BITS) + kvm_mmu_reset_context(vcpu); + + /* + * If CR4.PCIDE is changed 0 -> 1, there is no need to flush the TLB + * according to the SDM; however, stale prev_roots could be reused + * incorrectly in the future after a MOV to CR3 with NOFLUSH=1, so we + * free them all. This is *not* a superset of KVM_REQ_TLB_FLUSH_GUEST + * or KVM_REQ_TLB_FLUSH_CURRENT, because the hardware TLB is not flushed, + * so fall through. + */ + if (!tdp_enabled && + (cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) + kvm_mmu_unload(vcpu); + + /* + * The TLB has to be flushed for all PCIDs if any of the following + * (architecturally required) changes happen: + * - CR4.PCIDE is changed from 1 to 0 + * - CR4.PGE is toggled + * + * This is a superset of KVM_REQ_TLB_FLUSH_CURRENT. + */ + if (((cr4 ^ old_cr4) & X86_CR4_PGE) || + (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE))) + kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); + + /* + * The TLB has to be flushed for the current PCID if any of the + * following (architecturally required) changes happen: + * - CR4.SMEP is changed from 0 to 1 + * - CR4.PAE is toggled + */ + else if (((cr4 ^ old_cr4) & X86_CR4_PAE) || + ((cr4 & X86_CR4_SMEP) && !(old_cr4 & X86_CR4_SMEP))) + kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); + +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_post_set_cr4); + +int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) +{ + unsigned long old_cr4 = kvm_read_cr4(vcpu); + + if (!kvm_is_valid_cr4(vcpu, cr4)) + return 1; + + if (is_long_mode(vcpu)) { + if (!(cr4 & X86_CR4_PAE)) + return 1; + if ((cr4 ^ old_cr4) & X86_CR4_LA57) + return 1; + } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE) + && ((cr4 ^ old_cr4) & X86_CR4_PDPTR_BITS) + && !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) + return 1; + + if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) { + /* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */ + if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu)) + return 1; + } + + if ((cr4 & X86_CR4_CET) && !kvm_is_cr0_bit_set(vcpu, X86_CR0_WP)) + return 1; + + kvm_x86_call(set_cr4)(vcpu, cr4); + + kvm_post_set_cr4(vcpu, old_cr4, cr4); + + return 0; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr4); + +int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) +{ + if (cr8 & CR8_RESERVED_BITS) + return 1; + if (lapic_in_kernel(vcpu)) + kvm_lapic_set_tpr(vcpu, cr8); + else + vcpu->arch.cr8 = cr8; + return 0; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr8); + +unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu) +{ + if (lapic_in_kernel(vcpu)) + return kvm_lapic_get_cr8(vcpu); + else + return vcpu->arch.cr8; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_cr8); + +static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +{ + struct desc_ptr dt; + + if (vcpu->arch.guest_state_protected) + goto skip_protected_regs; + + kvm_handle_exception_payload_quirk(vcpu); + + kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); + kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); + kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES); + kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS); + kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS); + kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS); + + kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); + kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); + + kvm_x86_call(get_idt)(vcpu, &dt); + sregs->idt.limit = dt.size; + sregs->idt.base = dt.address; + kvm_x86_call(get_gdt)(vcpu, &dt); + sregs->gdt.limit = dt.size; + sregs->gdt.base = dt.address; + + sregs->cr2 = vcpu->arch.cr2; + sregs->cr3 = kvm_read_cr3(vcpu); + +skip_protected_regs: + sregs->cr0 = kvm_read_cr0(vcpu); + sregs->cr4 = kvm_read_cr4(vcpu); + sregs->cr8 = kvm_get_cr8(vcpu); + sregs->efer = vcpu->arch.efer; + sregs->apic_base = vcpu->arch.apic_base; +} + +static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +{ + __get_sregs_common(vcpu, sregs); + + if (vcpu->arch.guest_state_protected) + return; + + if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft) + set_bit(vcpu->arch.interrupt.nr, + (unsigned long *)sregs->interrupt_bitmap); +} + +int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + if (vcpu->kvm->arch.has_protected_state && + vcpu->arch.guest_state_protected) + return -EINVAL; + + vcpu_load(vcpu); + __get_sregs(vcpu, sregs); + vcpu_put(vcpu); + return 0; +} + +void kvm_vcpu_ioctl_x86_get_sregs2(struct kvm_vcpu *vcpu, + struct kvm_sregs2 *sregs2) +{ + int i; + + __get_sregs_common(vcpu, (struct kvm_sregs *)sregs2); + + if (vcpu->arch.guest_state_protected) + return; + + if (is_pae_paging(vcpu)) { + kvm_vcpu_srcu_read_lock(vcpu); + for (i = 0 ; i < 4 ; i++) + sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i); + sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID; + kvm_vcpu_srcu_read_unlock(vcpu); + } +} + +static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +{ + if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) { + /* + * When EFER.LME and CR0.PG are set, the processor is in + * 64-bit mode (though maybe in a 32-bit code segment). + * CR4.PAE and EFER.LMA must be set. + */ + if (!(sregs->cr4 & X86_CR4_PAE) || !(sregs->efer & EFER_LMA)) + return false; + if (!kvm_vcpu_is_legal_cr3(vcpu, sregs->cr3)) + return false; + } else { + /* + * Not in 64-bit mode: EFER.LMA is clear and the code + * segment cannot be 64-bit. + */ + if (sregs->efer & EFER_LMA || sregs->cs.l) + return false; + } + + return kvm_is_valid_cr4(vcpu, sregs->cr4) && + kvm_is_valid_cr0(vcpu, sregs->cr0) && + kvm_valid_efer(vcpu, sregs->efer); +} + +static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs, + int *mmu_reset_needed, bool update_pdptrs) +{ + int idx; + struct desc_ptr dt; + + if (!kvm_is_valid_sregs(vcpu, sregs)) + return -EINVAL; + + if (kvm_apic_set_base(vcpu, sregs->apic_base, true)) + return -EINVAL; + + if (vcpu->arch.guest_state_protected) + return 0; + + dt.size = sregs->idt.limit; + dt.address = sregs->idt.base; + kvm_x86_call(set_idt)(vcpu, &dt); + dt.size = sregs->gdt.limit; + dt.address = sregs->gdt.base; + kvm_x86_call(set_gdt)(vcpu, &dt); + + vcpu->arch.cr2 = sregs->cr2; + *mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3; + vcpu->arch.cr3 = sregs->cr3; + kvm_register_mark_dirty(vcpu, VCPU_REG_CR3); + kvm_x86_call(post_set_cr3)(vcpu, sregs->cr3); + + *mmu_reset_needed |= vcpu->arch.efer != sregs->efer; + kvm_x86_call(set_efer)(vcpu, sregs->efer); + + *mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0; + kvm_x86_call(set_cr0)(vcpu, sregs->cr0); + + *mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4; + kvm_x86_call(set_cr4)(vcpu, sregs->cr4); + + if (update_pdptrs) { + idx = srcu_read_lock(&vcpu->kvm->srcu); + if (is_pae_paging(vcpu)) { + load_pdptrs(vcpu, kvm_read_cr3(vcpu)); + *mmu_reset_needed = 1; + } + srcu_read_unlock(&vcpu->kvm->srcu, idx); + } + + kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); + kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); + kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES); + kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS); + kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS); + kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS); + + kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); + kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); + + kvm_set_cr8(vcpu, sregs->cr8); + + /* Older userspace won't unhalt the vcpu on reset. */ + if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 && + sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 && + !is_protmode(vcpu)) + kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); + + return 0; +} + +static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +{ + int pending_vec, max_bits; + int mmu_reset_needed = 0; + int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true); + + if (ret) + return ret; + + if (mmu_reset_needed) { + kvm_mmu_reset_context(vcpu); + kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); + } + + max_bits = KVM_NR_INTERRUPTS; + pending_vec = find_first_bit( + (const unsigned long *)sregs->interrupt_bitmap, max_bits); + + if (pending_vec < max_bits) { + kvm_queue_interrupt(vcpu, pending_vec, false); + pr_debug("Set back pending irq %d\n", pending_vec); + kvm_make_request(KVM_REQ_EVENT, vcpu); + } + return 0; +} + +int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + int ret; + + if (vcpu->kvm->arch.has_protected_state && + vcpu->arch.guest_state_protected) + return -EINVAL; + + vcpu_load(vcpu); + ret = __set_sregs(vcpu, sregs); + vcpu_put(vcpu); + return ret; +} + +int kvm_vcpu_ioctl_x86_set_sregs2(struct kvm_vcpu *vcpu, + struct kvm_sregs2 *sregs2) +{ + int mmu_reset_needed = 0; + bool valid_pdptrs = sregs2->flags & KVM_SREGS2_FLAGS_PDPTRS_VALID; + bool pae = (sregs2->cr0 & X86_CR0_PG) && (sregs2->cr4 & X86_CR4_PAE) && + !(sregs2->efer & EFER_LMA); + int i, ret; + + if (sregs2->flags & ~KVM_SREGS2_FLAGS_PDPTRS_VALID) + return -EINVAL; + + if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected)) + return -EINVAL; + + ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2, + &mmu_reset_needed, !valid_pdptrs); + if (ret) + return ret; + + if (valid_pdptrs) { + for (i = 0; i < 4 ; i++) + kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]); + + kvm_register_mark_dirty(vcpu, VCPU_REG_PDPTR); + mmu_reset_needed = 1; + vcpu->arch.pdptrs_from_userspace = true; + } + if (mmu_reset_needed) { + kvm_mmu_reset_context(vcpu); + kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); + } + return 0; +} + +void kvm_run_sync_regs_to_user(struct kvm_vcpu *vcpu) +{ + BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES); + + if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS) + __get_regs(vcpu, &vcpu->run->s.regs.regs); + + if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS) + __get_sregs(vcpu, &vcpu->run->s.regs.sregs); +} + +int kvm_run_sync_regs_from_user(struct kvm_vcpu *vcpu) +{ + if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) { + __set_regs(vcpu, &vcpu->run->s.regs.regs); + vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS; + } + + if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) { + struct kvm_sregs sregs = vcpu->run->s.regs.sregs; + + if (__set_sregs(vcpu, &sregs)) + return -EINVAL; + + vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS; + } + + return 0; +} + +void kvm_update_dr0123(struct kvm_vcpu *vcpu) +{ + int i; + + if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) { + for (i = 0; i < KVM_NR_DB_REGS; i++) + vcpu->arch.eff_db[i] = vcpu->arch.db[i]; + } +} + +void kvm_update_dr7(struct kvm_vcpu *vcpu) +{ + unsigned long dr7; + + if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) + dr7 = vcpu->arch.guest_debug_dr7; + else + dr7 = vcpu->arch.dr7; + kvm_x86_call(set_dr7)(vcpu, dr7); + vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED; + if (dr7 & DR7_BP_EN_MASK) + vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7); + +static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu) +{ + u64 fixed = DR6_FIXED_1; + + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_RTM)) + fixed |= DR6_RTM; + + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) + fixed |= DR6_BUS_LOCK; + return fixed; +} + +int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) +{ + size_t size = ARRAY_SIZE(vcpu->arch.db); + + switch (dr) { + case 0 ... 3: + vcpu->arch.db[array_index_nospec(dr, size)] = val; + if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) + vcpu->arch.eff_db[dr] = val; + break; + case 4: + case 6: + if (!kvm_dr6_valid(val)) + return 1; /* #GP */ + vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu); + break; + case 5: + default: /* 7 */ + if (!kvm_dr7_valid(val)) + return 1; /* #GP */ + vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1; + kvm_update_dr7(vcpu); + break; + } + + return 0; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_dr); + +unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr) +{ + size_t size = ARRAY_SIZE(vcpu->arch.db); + + switch (dr) { + case 0 ... 3: + return vcpu->arch.db[array_index_nospec(dr, size)]; + case 4: + case 6: + return vcpu->arch.dr6; + case 5: + default: /* 7 */ + return vcpu->arch.dr7; + } +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_dr); + +int kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu, + struct kvm_debugregs *dbgregs) +{ + unsigned int i; + + if (vcpu->kvm->arch.has_protected_state && + vcpu->arch.guest_state_protected) + return -EINVAL; + + kvm_handle_exception_payload_quirk(vcpu); + + memset(dbgregs, 0, sizeof(*dbgregs)); + + BUILD_BUG_ON(ARRAY_SIZE(vcpu->arch.db) != ARRAY_SIZE(dbgregs->db)); + for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++) + dbgregs->db[i] = vcpu->arch.db[i]; + + dbgregs->dr6 = vcpu->arch.dr6; + dbgregs->dr7 = vcpu->arch.dr7; + return 0; +} + +int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, + struct kvm_debugregs *dbgregs) +{ + unsigned int i; + + if (vcpu->kvm->arch.has_protected_state && + vcpu->arch.guest_state_protected) + return -EINVAL; + + if (dbgregs->flags) + return -EINVAL; + + if (!kvm_dr6_valid(dbgregs->dr6)) + return -EINVAL; + if (!kvm_dr7_valid(dbgregs->dr7)) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++) + vcpu->arch.db[i] = dbgregs->db[i]; + + kvm_update_dr0123(vcpu); + vcpu->arch.dr6 = dbgregs->dr6; + vcpu->arch.dr7 = dbgregs->dr7; + kvm_update_dr7(vcpu); + + return 0; +} diff --git a/arch/x86/kvm/regs.h b/arch/x86/kvm/regs.h index 5bda738afb7cad..28f62583545f79 100644 --- a/arch/x86/kvm/regs.h +++ b/arch/x86/kvm/regs.h @@ -397,6 +397,14 @@ static inline bool kvm_dr6_valid(u64 data) return !(data >> 32); } +static inline unsigned long kvm_get_effective_dr7(struct kvm_vcpu *vcpu) +{ + if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) + return vcpu->arch.guest_debug_dr7; + + return vcpu->arch.dr7; +} + static inline void enter_guest_mode(struct kvm_vcpu *vcpu) { vcpu->arch.hflags |= HF_GUEST_MASK; @@ -420,4 +428,26 @@ static inline bool is_guest_mode(struct kvm_vcpu *vcpu) return vcpu->arch.hflags & HF_GUEST_MASK; } +static inline unsigned long kvm_get_segment_base(struct kvm_vcpu *vcpu, int seg) +{ + return kvm_x86_call(get_segment_base)(vcpu, seg); +} + +void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); + +void kvm_vcpu_ioctl_x86_get_sregs2(struct kvm_vcpu *vcpu, + struct kvm_sregs2 *sregs2); +int kvm_vcpu_ioctl_x86_set_sregs2(struct kvm_vcpu *vcpu, + struct kvm_sregs2 *sregs2); + +void kvm_run_sync_regs_to_user(struct kvm_vcpu *vcpu); +int kvm_run_sync_regs_from_user(struct kvm_vcpu *vcpu); + +void kvm_update_dr0123(struct kvm_vcpu *vcpu); +int kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu, + struct kvm_debugregs *dbgregs); +int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, + struct kvm_debugregs *dbgregs); + + #endif diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index cb94b87df98989..9822ff0450bb2d 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -129,13 +129,9 @@ static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE); KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST) static void process_nmi(struct kvm_vcpu *vcpu); -static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); static void store_regs(struct kvm_vcpu *vcpu); static int sync_regs(struct kvm_vcpu *vcpu); -static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2); -static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2); - static DEFINE_MUTEX(vendor_module_lock); static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu); static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu); @@ -1017,170 +1013,6 @@ bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_require_dr); -static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2); -} - -/* - * Load the pae pdptrs. Return 1 if they are all valid, 0 otherwise. - */ -int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) -{ - struct kvm_mmu *mmu = vcpu->arch.walk_mmu; - gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; - gpa_t real_gpa; - int i; - int ret; - u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)]; - - /* - * If the MMU is nested, CR3 holds an L2 GPA and needs to be translated - * to an L1 GPA. - */ - real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn), - PFERR_USER_MASK | PFERR_WRITE_MASK | - PFERR_GUEST_PAGE_MASK, NULL, 0); - if (real_gpa == INVALID_GPA) - return 0; - - /* Note the offset, PDPTRs are 32 byte aligned when using PAE paging. */ - ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte, - cr3 & GENMASK(11, 5), sizeof(pdpte)); - if (ret < 0) - return 0; - - for (i = 0; i < ARRAY_SIZE(pdpte); ++i) { - if ((pdpte[i] & PT_PRESENT_MASK) && - (pdpte[i] & pdptr_rsvd_bits(vcpu))) { - return 0; - } - } - - /* - * Marking VCPU_REG_PDPTR dirty doesn't work for !tdp_enabled. - * Shadow page roots need to be reconstructed instead. - */ - if (!tdp_enabled && memcmp(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs))) - kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT); - - memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs)); - kvm_register_mark_dirty(vcpu, VCPU_REG_PDPTR); - kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); - vcpu->arch.pdptrs_from_userspace = false; - - return 1; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(load_pdptrs); - -static bool kvm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) -{ -#ifdef CONFIG_X86_64 - if (cr0 & 0xffffffff00000000UL) - return false; -#endif - - if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) - return false; - - if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) - return false; - - return kvm_x86_call(is_valid_cr0)(vcpu, cr0); -} - -void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0) -{ - /* - * CR0.WP is incorporated into the MMU role, but only for non-nested, - * indirect shadow MMUs. If paging is disabled, no updates are needed - * as there are no permission bits to emulate. If TDP is enabled, the - * MMU's metadata needs to be updated, e.g. so that emulating guest - * translations does the right thing, but there's no need to unload the - * root as CR0.WP doesn't affect SPTEs. - */ - if ((cr0 ^ old_cr0) == X86_CR0_WP) { - if (!(cr0 & X86_CR0_PG)) - return; - - if (tdp_enabled) { - kvm_init_mmu(vcpu); - return; - } - } - - if ((cr0 ^ old_cr0) & X86_CR0_PG) { - /* - * Clearing CR0.PG is defined to flush the TLB from the guest's - * perspective. - */ - if (!(cr0 & X86_CR0_PG)) - kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); - /* - * Check for async #PF completion events when enabling paging, - * as the vCPU may have previously encountered async #PFs (it's - * entirely legal for the guest to toggle paging on/off without - * waiting for the async #PF queue to drain). - */ - else if (kvm_pv_async_pf_enabled(vcpu)) - kvm_make_request(KVM_REQ_APF_READY, vcpu); - } - - if ((cr0 ^ old_cr0) & KVM_MMU_CR0_ROLE_BITS) - kvm_mmu_reset_context(vcpu); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_post_set_cr0); - -int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) -{ - unsigned long old_cr0 = kvm_read_cr0(vcpu); - - if (!kvm_is_valid_cr0(vcpu, cr0)) - return 1; - - cr0 |= X86_CR0_ET; - - /* Write to CR0 reserved bits are ignored, even on Intel. */ - cr0 &= ~CR0_RESERVED_BITS; - -#ifdef CONFIG_X86_64 - if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) && - (cr0 & X86_CR0_PG)) { - int cs_db, cs_l; - - if (!is_pae(vcpu)) - return 1; - kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); - if (cs_l) - return 1; - } -#endif - if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) && - is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) && - !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) - return 1; - - if (!(cr0 & X86_CR0_PG) && - (is_64_bit_mode(vcpu) || kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE))) - return 1; - - if (!(cr0 & X86_CR0_WP) && kvm_is_cr4_bit_set(vcpu, X86_CR4_CET)) - return 1; - - kvm_x86_call(set_cr0)(vcpu, cr0); - - kvm_post_set_cr0(vcpu, old_cr0, cr0); - - return 0; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr0); - -void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw) -{ - (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f)); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_lmsw); - static void kvm_load_xfeatures(struct kvm_vcpu *vcpu, bool load_guest) { if (vcpu->arch.guest_state_protected) @@ -1290,89 +1122,7 @@ int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_xsetbv); -static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) -{ - return __kvm_is_valid_cr4(vcpu, cr4) && - kvm_x86_call(is_valid_cr4)(vcpu, cr4); -} - -void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4) -{ - if ((cr4 ^ old_cr4) & KVM_MMU_CR4_ROLE_BITS) - kvm_mmu_reset_context(vcpu); - - /* - * If CR4.PCIDE is changed 0 -> 1, there is no need to flush the TLB - * according to the SDM; however, stale prev_roots could be reused - * incorrectly in the future after a MOV to CR3 with NOFLUSH=1, so we - * free them all. This is *not* a superset of KVM_REQ_TLB_FLUSH_GUEST - * or KVM_REQ_TLB_FLUSH_CURRENT, because the hardware TLB is not flushed, - * so fall through. - */ - if (!tdp_enabled && - (cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) - kvm_mmu_unload(vcpu); - - /* - * The TLB has to be flushed for all PCIDs if any of the following - * (architecturally required) changes happen: - * - CR4.PCIDE is changed from 1 to 0 - * - CR4.PGE is toggled - * - * This is a superset of KVM_REQ_TLB_FLUSH_CURRENT. - */ - if (((cr4 ^ old_cr4) & X86_CR4_PGE) || - (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE))) - kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); - - /* - * The TLB has to be flushed for the current PCID if any of the - * following (architecturally required) changes happen: - * - CR4.SMEP is changed from 0 to 1 - * - CR4.PAE is toggled - */ - else if (((cr4 ^ old_cr4) & X86_CR4_PAE) || - ((cr4 & X86_CR4_SMEP) && !(old_cr4 & X86_CR4_SMEP))) - kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); - -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_post_set_cr4); - -int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) -{ - unsigned long old_cr4 = kvm_read_cr4(vcpu); - - if (!kvm_is_valid_cr4(vcpu, cr4)) - return 1; - - if (is_long_mode(vcpu)) { - if (!(cr4 & X86_CR4_PAE)) - return 1; - if ((cr4 ^ old_cr4) & X86_CR4_LA57) - return 1; - } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE) - && ((cr4 ^ old_cr4) & X86_CR4_PDPTR_BITS) - && !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) - return 1; - - if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) { - /* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */ - if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu)) - return 1; - } - - if ((cr4 & X86_CR4_CET) && !kvm_is_cr0_bit_set(vcpu, X86_CR0_WP)) - return 1; - - kvm_x86_call(set_cr4)(vcpu, cr4); - - kvm_post_set_cr4(vcpu, old_cr4, cr4); - - return 0; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr4); - -static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid) +void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid) { struct kvm_mmu *mmu = vcpu->arch.mmu; unsigned long roots_to_free = 0; @@ -1415,167 +1165,6 @@ static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid) kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free); } -int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) -{ - bool skip_tlb_flush = false; - unsigned long pcid = 0; -#ifdef CONFIG_X86_64 - if (kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)) { - skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH; - cr3 &= ~X86_CR3_PCID_NOFLUSH; - pcid = cr3 & X86_CR3_PCID_MASK; - } -#endif - - /* PDPTRs are always reloaded for PAE paging. */ - if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu)) - goto handle_tlb_flush; - - /* - * Do not condition the GPA check on long mode, this helper is used to - * stuff CR3, e.g. for RSM emulation, and there is no guarantee that - * the current vCPU mode is accurate. - */ - if (!kvm_vcpu_is_legal_cr3(vcpu, cr3)) - return 1; - - if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3)) - return 1; - - if (cr3 != kvm_read_cr3(vcpu)) - kvm_mmu_new_pgd(vcpu, cr3); - - vcpu->arch.cr3 = cr3; - kvm_register_mark_dirty(vcpu, VCPU_REG_CR3); - /* Do not call post_set_cr3, we do not get here for confidential guests. */ - -handle_tlb_flush: - /* - * A load of CR3 that flushes the TLB flushes only the current PCID, - * even if PCID is disabled, in which case PCID=0 is flushed. It's a - * moot point in the end because _disabling_ PCID will flush all PCIDs, - * and it's impossible to use a non-zero PCID when PCID is disabled, - * i.e. only PCID=0 can be relevant. - */ - if (!skip_tlb_flush) - kvm_invalidate_pcid(vcpu, pcid); - - return 0; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr3); - -int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) -{ - if (cr8 & CR8_RESERVED_BITS) - return 1; - if (lapic_in_kernel(vcpu)) - kvm_lapic_set_tpr(vcpu, cr8); - else - vcpu->arch.cr8 = cr8; - return 0; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr8); - -unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu) -{ - if (lapic_in_kernel(vcpu)) - return kvm_lapic_get_cr8(vcpu); - else - return vcpu->arch.cr8; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_cr8); - -static void kvm_update_dr0123(struct kvm_vcpu *vcpu) -{ - int i; - - if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) { - for (i = 0; i < KVM_NR_DB_REGS; i++) - vcpu->arch.eff_db[i] = vcpu->arch.db[i]; - } -} - -void kvm_update_dr7(struct kvm_vcpu *vcpu) -{ - unsigned long dr7; - - if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) - dr7 = vcpu->arch.guest_debug_dr7; - else - dr7 = vcpu->arch.dr7; - kvm_x86_call(set_dr7)(vcpu, dr7); - vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED; - if (dr7 & DR7_BP_EN_MASK) - vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7); - -static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu) -{ - u64 fixed = DR6_FIXED_1; - - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_RTM)) - fixed |= DR6_RTM; - - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) - fixed |= DR6_BUS_LOCK; - return fixed; -} - -int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) -{ - size_t size = ARRAY_SIZE(vcpu->arch.db); - - switch (dr) { - case 0 ... 3: - vcpu->arch.db[array_index_nospec(dr, size)] = val; - if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) - vcpu->arch.eff_db[dr] = val; - break; - case 4: - case 6: - if (!kvm_dr6_valid(val)) - return 1; /* #GP */ - vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu); - break; - case 5: - default: /* 7 */ - if (!kvm_dr7_valid(val)) - return 1; /* #GP */ - vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1; - kvm_update_dr7(vcpu); - break; - } - - return 0; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_dr); - -unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr) -{ - size_t size = ARRAY_SIZE(vcpu->arch.db); - - switch (dr) { - case 0 ... 3: - return vcpu->arch.db[array_index_nospec(dr, size)]; - case 4: - case 6: - return vcpu->arch.dr6; - case 5: - default: /* 7 */ - return vcpu->arch.dr7; - } -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_dr); - -static unsigned long kvm_get_effective_dr7(struct kvm_vcpu *vcpu) -{ - if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) - return vcpu->arch.guest_debug_dr7; - - return vcpu->arch.dr7; -} - int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu) { u32 pmc = kvm_ecx_read(vcpu); @@ -5532,7 +5121,7 @@ static struct kvm_queued_exception *kvm_get_exception_to_save(struct kvm_vcpu *v return &vcpu->arch.exception; } -static void kvm_handle_exception_payload_quirk(struct kvm_vcpu *vcpu) +void kvm_handle_exception_payload_quirk(struct kvm_vcpu *vcpu) { struct kvm_queued_exception *ex = kvm_get_exception_to_save(vcpu); @@ -5736,57 +5325,6 @@ static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, return 0; } -static int kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu, - struct kvm_debugregs *dbgregs) -{ - unsigned int i; - - if (vcpu->kvm->arch.has_protected_state && - vcpu->arch.guest_state_protected) - return -EINVAL; - - kvm_handle_exception_payload_quirk(vcpu); - - memset(dbgregs, 0, sizeof(*dbgregs)); - - BUILD_BUG_ON(ARRAY_SIZE(vcpu->arch.db) != ARRAY_SIZE(dbgregs->db)); - for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++) - dbgregs->db[i] = vcpu->arch.db[i]; - - dbgregs->dr6 = vcpu->arch.dr6; - dbgregs->dr7 = vcpu->arch.dr7; - return 0; -} - -static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, - struct kvm_debugregs *dbgregs) -{ - unsigned int i; - - if (vcpu->kvm->arch.has_protected_state && - vcpu->arch.guest_state_protected) - return -EINVAL; - - if (dbgregs->flags) - return -EINVAL; - - if (!kvm_dr6_valid(dbgregs->dr6)) - return -EINVAL; - if (!kvm_dr7_valid(dbgregs->dr7)) - return -EINVAL; - - for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++) - vcpu->arch.db[i] = dbgregs->db[i]; - - kvm_update_dr0123(vcpu); - vcpu->arch.dr6 = dbgregs->dr6; - vcpu->arch.dr7 = dbgregs->dr7; - kvm_update_dr7(vcpu); - - return 0; -} - - static int kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu *vcpu, u8 *state, unsigned int size) { @@ -6623,7 +6161,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, r = -ENOMEM; if (!u.sregs2) goto out; - __get_sregs2(vcpu, u.sregs2); + kvm_vcpu_ioctl_x86_get_sregs2(vcpu, u.sregs2); r = -EFAULT; if (copy_to_user(argp, u.sregs2, sizeof(struct kvm_sregs2))) goto out; @@ -6642,7 +6180,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, u.sregs2 = NULL; goto out; } - r = __set_sregs2(vcpu, u.sregs2); + r = kvm_vcpu_ioctl_x86_set_sregs2(vcpu, u.sregs2); break; } case KVM_HAS_DEVICE_ATTR: @@ -8495,11 +8033,6 @@ static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt, return emulator_pio_out(emul_to_vcpu(ctxt), size, port, val, count); } -static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg) -{ - return kvm_x86_call(get_segment_base)(vcpu, seg); -} - static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address) { kvm_mmu_invlpg(emul_to_vcpu(ctxt), address); @@ -8644,7 +8177,7 @@ static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) static unsigned long emulator_get_cached_segment_base( struct x86_emulate_ctxt *ctxt, int seg) { - return get_segment_base(emul_to_vcpu(ctxt), seg); + return kvm_get_segment_base(emul_to_vcpu(ctxt), seg); } static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector, @@ -12084,179 +11617,6 @@ out: return r; } -static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) -{ - if (vcpu->arch.emulate_regs_need_sync_to_vcpu) { - /* - * We are here if userspace calls get_regs() in the middle of - * instruction emulation. Registers state needs to be copied - * back from emulation context to vcpu. Userspace shouldn't do - * that usually, but some bad designed PV devices (vmware - * backdoor interface) need this to work - */ - emulator_writeback_register_cache(vcpu->arch.emulate_ctxt); - vcpu->arch.emulate_regs_need_sync_to_vcpu = false; - } - regs->rax = kvm_rax_read_raw(vcpu); - regs->rbx = kvm_rbx_read_raw(vcpu); - regs->rcx = kvm_rcx_read_raw(vcpu); - regs->rdx = kvm_rdx_read_raw(vcpu); - regs->rsi = kvm_rsi_read_raw(vcpu); - regs->rdi = kvm_rdi_read_raw(vcpu); - regs->rsp = kvm_rsp_read(vcpu); - regs->rbp = kvm_rbp_read_raw(vcpu); -#ifdef CONFIG_X86_64 - regs->r8 = kvm_r8_read_raw(vcpu); - regs->r9 = kvm_r9_read_raw(vcpu); - regs->r10 = kvm_r10_read_raw(vcpu); - regs->r11 = kvm_r11_read_raw(vcpu); - regs->r12 = kvm_r12_read_raw(vcpu); - regs->r13 = kvm_r13_read_raw(vcpu); - regs->r14 = kvm_r14_read_raw(vcpu); - regs->r15 = kvm_r15_read_raw(vcpu); -#endif - - regs->rip = kvm_rip_read(vcpu); - regs->rflags = kvm_get_rflags(vcpu); -} - -int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) -{ - if (vcpu->kvm->arch.has_protected_state && - vcpu->arch.guest_state_protected) - return -EINVAL; - - vcpu_load(vcpu); - __get_regs(vcpu, regs); - vcpu_put(vcpu); - return 0; -} - -static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) -{ - vcpu->arch.emulate_regs_need_sync_from_vcpu = true; - vcpu->arch.emulate_regs_need_sync_to_vcpu = false; - - kvm_rax_write_raw(vcpu, regs->rax); - kvm_rbx_write_raw(vcpu, regs->rbx); - kvm_rcx_write_raw(vcpu, regs->rcx); - kvm_rdx_write_raw(vcpu, regs->rdx); - kvm_rsi_write_raw(vcpu, regs->rsi); - kvm_rdi_write_raw(vcpu, regs->rdi); - kvm_rsp_write(vcpu, regs->rsp); - kvm_rbp_write_raw(vcpu, regs->rbp); -#ifdef CONFIG_X86_64 - kvm_r8_write_raw(vcpu, regs->r8); - kvm_r9_write_raw(vcpu, regs->r9); - kvm_r10_write_raw(vcpu, regs->r10); - kvm_r11_write_raw(vcpu, regs->r11); - kvm_r12_write_raw(vcpu, regs->r12); - kvm_r13_write_raw(vcpu, regs->r13); - kvm_r14_write_raw(vcpu, regs->r14); - kvm_r15_write_raw(vcpu, regs->r15); -#endif - - kvm_rip_write(vcpu, regs->rip); - kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED); - - vcpu->arch.exception.pending = false; - vcpu->arch.exception_vmexit.pending = false; - - kvm_make_request(KVM_REQ_EVENT, vcpu); -} - -int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) -{ - if (vcpu->kvm->arch.has_protected_state && - vcpu->arch.guest_state_protected) - return -EINVAL; - - vcpu_load(vcpu); - __set_regs(vcpu, regs); - vcpu_put(vcpu); - return 0; -} - -static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) -{ - struct desc_ptr dt; - - if (vcpu->arch.guest_state_protected) - goto skip_protected_regs; - - kvm_handle_exception_payload_quirk(vcpu); - - kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); - kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); - kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES); - kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS); - kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS); - kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS); - - kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); - kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); - - kvm_x86_call(get_idt)(vcpu, &dt); - sregs->idt.limit = dt.size; - sregs->idt.base = dt.address; - kvm_x86_call(get_gdt)(vcpu, &dt); - sregs->gdt.limit = dt.size; - sregs->gdt.base = dt.address; - - sregs->cr2 = vcpu->arch.cr2; - sregs->cr3 = kvm_read_cr3(vcpu); - -skip_protected_regs: - sregs->cr0 = kvm_read_cr0(vcpu); - sregs->cr4 = kvm_read_cr4(vcpu); - sregs->cr8 = kvm_get_cr8(vcpu); - sregs->efer = vcpu->arch.efer; - sregs->apic_base = vcpu->arch.apic_base; -} - -static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) -{ - __get_sregs_common(vcpu, sregs); - - if (vcpu->arch.guest_state_protected) - return; - - if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft) - set_bit(vcpu->arch.interrupt.nr, - (unsigned long *)sregs->interrupt_bitmap); -} - -static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2) -{ - int i; - - __get_sregs_common(vcpu, (struct kvm_sregs *)sregs2); - - if (vcpu->arch.guest_state_protected) - return; - - if (is_pae_paging(vcpu)) { - kvm_vcpu_srcu_read_lock(vcpu); - for (i = 0 ; i < 4 ; i++) - sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i); - sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID; - kvm_vcpu_srcu_read_unlock(vcpu); - } -} - -int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, - struct kvm_sregs *sregs) -{ - if (vcpu->kvm->arch.has_protected_state && - vcpu->arch.guest_state_protected) - return -EINVAL; - - vcpu_load(vcpu); - __get_sregs(vcpu, sregs); - vcpu_put(vcpu); - return 0; -} - int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, struct kvm_mp_state *mp_state) { @@ -12376,173 +11736,6 @@ unhandled_task_switch: } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_task_switch); -static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) -{ - if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) { - /* - * When EFER.LME and CR0.PG are set, the processor is in - * 64-bit mode (though maybe in a 32-bit code segment). - * CR4.PAE and EFER.LMA must be set. - */ - if (!(sregs->cr4 & X86_CR4_PAE) || !(sregs->efer & EFER_LMA)) - return false; - if (!kvm_vcpu_is_legal_cr3(vcpu, sregs->cr3)) - return false; - } else { - /* - * Not in 64-bit mode: EFER.LMA is clear and the code - * segment cannot be 64-bit. - */ - if (sregs->efer & EFER_LMA || sregs->cs.l) - return false; - } - - return kvm_is_valid_cr4(vcpu, sregs->cr4) && - kvm_is_valid_cr0(vcpu, sregs->cr0); -} - -static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs, - int *mmu_reset_needed, bool update_pdptrs) -{ - int idx; - struct desc_ptr dt; - - if (!kvm_is_valid_sregs(vcpu, sregs)) - return -EINVAL; - - if (kvm_apic_set_base(vcpu, sregs->apic_base, true)) - return -EINVAL; - - if (vcpu->arch.guest_state_protected) - return 0; - - dt.size = sregs->idt.limit; - dt.address = sregs->idt.base; - kvm_x86_call(set_idt)(vcpu, &dt); - dt.size = sregs->gdt.limit; - dt.address = sregs->gdt.base; - kvm_x86_call(set_gdt)(vcpu, &dt); - - vcpu->arch.cr2 = sregs->cr2; - *mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3; - vcpu->arch.cr3 = sregs->cr3; - kvm_register_mark_dirty(vcpu, VCPU_REG_CR3); - kvm_x86_call(post_set_cr3)(vcpu, sregs->cr3); - - *mmu_reset_needed |= vcpu->arch.efer != sregs->efer; - kvm_x86_call(set_efer)(vcpu, sregs->efer); - - *mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0; - kvm_x86_call(set_cr0)(vcpu, sregs->cr0); - - *mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4; - kvm_x86_call(set_cr4)(vcpu, sregs->cr4); - - if (update_pdptrs) { - idx = srcu_read_lock(&vcpu->kvm->srcu); - if (is_pae_paging(vcpu)) { - load_pdptrs(vcpu, kvm_read_cr3(vcpu)); - *mmu_reset_needed = 1; - } - srcu_read_unlock(&vcpu->kvm->srcu, idx); - } - - kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); - kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); - kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES); - kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS); - kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS); - kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS); - - kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); - kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); - - kvm_set_cr8(vcpu, sregs->cr8); - - /* Older userspace won't unhalt the vcpu on reset. */ - if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 && - sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 && - !is_protmode(vcpu)) - kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); - - return 0; -} - -static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) -{ - int pending_vec, max_bits; - int mmu_reset_needed = 0; - int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true); - - if (ret) - return ret; - - if (mmu_reset_needed) { - kvm_mmu_reset_context(vcpu); - kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); - } - - max_bits = KVM_NR_INTERRUPTS; - pending_vec = find_first_bit( - (const unsigned long *)sregs->interrupt_bitmap, max_bits); - - if (pending_vec < max_bits) { - kvm_queue_interrupt(vcpu, pending_vec, false); - pr_debug("Set back pending irq %d\n", pending_vec); - kvm_make_request(KVM_REQ_EVENT, vcpu); - } - return 0; -} - -static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2) -{ - int mmu_reset_needed = 0; - bool valid_pdptrs = sregs2->flags & KVM_SREGS2_FLAGS_PDPTRS_VALID; - bool pae = (sregs2->cr0 & X86_CR0_PG) && (sregs2->cr4 & X86_CR4_PAE) && - !(sregs2->efer & EFER_LMA); - int i, ret; - - if (sregs2->flags & ~KVM_SREGS2_FLAGS_PDPTRS_VALID) - return -EINVAL; - - if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected)) - return -EINVAL; - - ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2, - &mmu_reset_needed, !valid_pdptrs); - if (ret) - return ret; - - if (valid_pdptrs) { - for (i = 0; i < 4 ; i++) - kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]); - - kvm_register_mark_dirty(vcpu, VCPU_REG_PDPTR); - mmu_reset_needed = 1; - vcpu->arch.pdptrs_from_userspace = true; - } - if (mmu_reset_needed) { - kvm_mmu_reset_context(vcpu); - kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); - } - return 0; -} - -int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, - struct kvm_sregs *sregs) -{ - int ret; - - if (vcpu->kvm->arch.has_protected_state && - vcpu->arch.guest_state_protected) - return -EINVAL; - - vcpu_load(vcpu); - ret = __set_sregs(vcpu, sregs); - vcpu_put(vcpu); - return ret; -} - static void kvm_arch_vcpu_guestdbg_update_apicv_inhibit(struct kvm *kvm) { bool set = false; @@ -12698,13 +11891,7 @@ int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) static void store_regs(struct kvm_vcpu *vcpu) { - BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES); - - if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS) - __get_regs(vcpu, &vcpu->run->s.regs.regs); - - if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS) - __get_sregs(vcpu, &vcpu->run->s.regs.sregs); + kvm_run_sync_regs_to_user(vcpu); if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS) kvm_vcpu_ioctl_x86_get_vcpu_events( @@ -12713,19 +11900,8 @@ static void store_regs(struct kvm_vcpu *vcpu) static int sync_regs(struct kvm_vcpu *vcpu) { - if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) { - __set_regs(vcpu, &vcpu->run->s.regs.regs); - vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS; - } - - if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) { - struct kvm_sregs sregs = vcpu->run->s.regs.sregs; - - if (__set_sregs(vcpu, &sregs)) - return -EINVAL; - - vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS; - } + if (kvm_run_sync_regs_from_user(vcpu)) + return -EINVAL; if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) { struct kvm_vcpu_events events = vcpu->run->s.regs.events; @@ -13833,51 +13009,6 @@ int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu) return kvm_x86_call(interrupt_allowed)(vcpu, false); } -unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu) -{ - /* Can't read the RIP when guest state is protected, just return 0 */ - if (vcpu->arch.guest_state_protected) - return 0; - - if (is_64_bit_mode(vcpu)) - return kvm_rip_read(vcpu); - return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) + - kvm_rip_read(vcpu)); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_linear_rip); - -bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip) -{ - return kvm_get_linear_rip(vcpu) == linear_rip; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_is_linear_rip); - -unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu) -{ - unsigned long rflags; - - rflags = kvm_x86_call(get_rflags)(vcpu); - if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) - rflags &= ~X86_EFLAGS_TF; - return rflags; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_rflags); - -static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) -{ - if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP && - kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip)) - rflags |= X86_EFLAGS_TF; - kvm_x86_call(set_rflags)(vcpu, rflags); -} - -void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) -{ - __kvm_set_rflags(vcpu, rflags); - kvm_make_request(KVM_REQ_EVENT, vcpu); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_rflags); - static inline u32 kvm_async_pf_hash_fn(gfn_t gfn) { BUILD_BUG_ON(!is_power_of_2(ASYNC_PF_PER_VCPU)); diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index ad9d2aacb87d2c..bd2699bd3fe859 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -403,6 +403,7 @@ int handle_ud(struct kvm_vcpu *vcpu); void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, struct kvm_queued_exception *ex); +void kvm_handle_exception_payload_quirk(struct kvm_vcpu *vcpu); int kvm_mtrr_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data); int kvm_mtrr_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata); @@ -597,6 +598,7 @@ static inline void kvm_machine_check(void) int kvm_spec_ctrl_test_value(u64 value); int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r, struct x86_exception *e); +void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid); int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva); bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type); diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index d023a40a1e0345..3b7e807e803c36 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -418,6 +418,8 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa) int collapsed = 0; int i; + spin_lock(&cpa_lock); + if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) { for (i = 0; i < cpa->numpages; i++) collapsed += collapse_large_pages(__cpa_addr(cpa, i), @@ -431,8 +433,10 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa) collapsed += collapse_large_pages(addr, &pgtables); } - if (!collapsed) + if (!collapsed) { + spin_unlock(&cpa_lock); return; + } flush_tlb_all(); @@ -440,6 +444,8 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa) list_del(&ptdesc->pt_list); pagetable_free(ptdesc); } + + spin_unlock(&cpa_lock); } static void cpa_flush(struct cpa_data *cpa, int cache) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 276d076d299381..ea74ff1d166cdf 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1473,17 +1473,20 @@ static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size, * * Bit layout of `fixup` (32-bit): * - * +-----------+--------+-----------+---------+----------+ - * | 31 | 30-24 | 23-16 | 15-8 | 7-0 | - * | | | | | | - * | ARENA_ACC | Unused | ARENA_REG | DST_REG | INSN_LEN | - * +-----------+--------+-----------+---------+----------+ + * +-----------+-------------+--------+-----------+---------+----------+ + * | 31 | 30 | 29-24 | 23-16 | 15-8 | 7-0 | + * | | | | | | | + * | ARENA_ACC | ARENA_WRITE | Unused | ARENA_REG | DST_REG | INSN_LEN | + * +-----------+-------------+--------+-----------+---------+----------+ * * - INSN_LEN (8 bits): Length of faulting insn (max x86 insn = 15 bytes (fits in 8 bits)). * - DST_REG (8 bits): Offset of dst_reg from reg2pt_regs[] (max offset = 112 (fits in 8 bits)). - * This is set to DONT_CLEAR if the insn is a store. + * This is set to DONT_CLEAR if the insn does not read into a register. * - ARENA_REG (8 bits): Offset of the register that is used to calculate the * address for load/store when accessing the arena region. + * - ARENA_WRITE (1 bit): This bit is set when the faulting instruction wrote to the arena region. + * It is independent of DST_REG, since a read-modify-write both writes to + * memory and reads the old value into a register. * - ARENA_ACCESS (1 bit): This bit is set when the faulting instruction accessed the arena region. * * Bit layout of `data` (32-bit): @@ -1502,6 +1505,7 @@ static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size, #define FIXUP_INSN_LEN_MASK GENMASK(7, 0) #define FIXUP_REG_MASK GENMASK(15, 8) #define FIXUP_ARENA_REG_MASK GENMASK(23, 16) +#define FIXUP_ARENA_WRITE BIT(30) #define FIXUP_ARENA_ACCESS BIT(31) #define DATA_ARENA_OFFSET_MASK GENMASK(31, 16) @@ -1510,7 +1514,7 @@ bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs) u32 reg = FIELD_GET(FIXUP_REG_MASK, x->fixup); u32 insn_len = FIELD_GET(FIXUP_INSN_LEN_MASK, x->fixup); bool is_arena = !!(x->fixup & FIXUP_ARENA_ACCESS); - bool is_write = (reg == DONT_CLEAR); + bool is_write = !!(x->fixup & FIXUP_ARENA_WRITE); unsigned long addr; s16 off; u32 arena_reg; @@ -2304,6 +2308,7 @@ populate_extable: struct exception_table_entry *ex; u8 *_insn = image + proglen + (start_of_ldx - temp); u32 arena_reg, fixup_reg; + bool is_write; s64 delta; if (!bpf_prog->aux->extable) @@ -2331,19 +2336,38 @@ populate_extable: * BPF_PROBE_ATOMIC) before being used for the memory access. Pass * the reg holding the unmodified 32-bit address to * ex_handler_bpf(). + * + * A load-acquire is of BPF_STX class, but reads from src_reg + * into dst_reg like a BPF_LDX does, hence it must not be + * treated as a store here. */ - if (BPF_CLASS(insn->code) == BPF_LDX) { + if (BPF_CLASS(insn->code) == BPF_LDX || + bpf_atomic_is_load_acq(insn)) { arena_reg = reg2pt_regs[src_reg]; fixup_reg = reg2pt_regs[dst_reg]; + is_write = false; } else { + /* + * A store has no destination register to clear, + * except for a read-modify-write with BPF_FETCH, + * which also reads the old value into src_reg, or + * into r0 for a BPF_CMPXCHG. Either way the access + * is still reported as a write. + */ + int load_reg = bpf_atomic_load_reg(insn); + arena_reg = reg2pt_regs[dst_reg]; - fixup_reg = DONT_CLEAR; + fixup_reg = load_reg < 0 ? DONT_CLEAR : + reg2pt_regs[load_reg]; + is_write = true; } ex->fixup = FIELD_PREP(FIXUP_INSN_LEN_MASK, prog - start_of_ldx) | FIELD_PREP(FIXUP_ARENA_REG_MASK, arena_reg) | FIELD_PREP(FIXUP_REG_MASK, fixup_reg); ex->fixup |= FIXUP_ARENA_ACCESS; + if (is_write) + ex->fixup |= FIXUP_ARENA_WRITE; ex->data |= FIELD_PREP(DATA_ARENA_OFFSET_MASK, insn->off); } @@ -3369,11 +3393,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im WARN_ON_ONCE((flags & BPF_TRAMP_F_INDIRECT) && (flags & ~(BPF_TRAMP_F_INDIRECT | BPF_TRAMP_F_RET_FENTRY_RET))); - /* extra registers for struct arguments */ - for (i = 0; i < m->nr_args; i++) { - if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) - nr_regs += (m->arg_size[i] + 7) / 8 - 1; - } + for (i = 0; i < m->nr_args; i++) + nr_regs += (m->arg_size[i] + 7) / 8 - 1; /* x86-64 supports up to MAX_BPF_FUNC_ARGS arguments. 1-6 * are passed through regs, the remains are through stack. diff --git a/block/bio-integrity-auto.c b/block/bio-integrity-auto.c index b1c733ecfd2e8c..9456dcffd17a6a 100644 --- a/block/bio-integrity-auto.c +++ b/block/bio-integrity-auto.c @@ -44,12 +44,6 @@ static void bio_integrity_verify_fn(struct work_struct *work) bio_endio(bio); } -#define BIP_CHECK_FLAGS (BIP_CHECK_GUARD | BIP_CHECK_REFTAG | BIP_CHECK_APPTAG) -static bool bip_should_check(struct bio_integrity_payload *bip) -{ - return bip->bip_flags & BIP_CHECK_FLAGS; -} - /** * __bio_integrity_endio - Integrity I/O completion function * @bio: Protected bio @@ -66,7 +60,7 @@ bool __bio_integrity_endio(struct bio *bio) container_of(bip, struct bio_integrity_data, bip); if (bio_op(bio) == REQ_OP_READ && !bio->bi_status && - bip_should_check(bip)) { + (bip->bip_flags & BIP_CHECK_FLAGS)) { INIT_WORK(&bid->work, bio_integrity_verify_fn); queue_work(kintegrityd_wq, &bid->work); return false; @@ -99,7 +93,7 @@ void bio_integrity_prep(struct bio *bio, unsigned int action) bio_integrity_setup_default(bio); /* Auto-generate integrity metadata if this is a write */ - if (bio_data_dir(bio) == WRITE && bip_should_check(&bid->bip)) + if (bio_data_dir(bio) == WRITE && (bid->bip.bip_flags & BIP_CHECK_FLAGS)) bio_integrity_generate(bio); else bid->saved_bio_iter = bio->bi_iter; diff --git a/block/bio-integrity-fs.c b/block/bio-integrity-fs.c index 9c5fe5fa8f0dad..692403dfa047bb 100644 --- a/block/bio-integrity-fs.c +++ b/block/bio-integrity-fs.c @@ -46,7 +46,8 @@ void fs_bio_integrity_free(struct bio *bio) void fs_bio_integrity_generate(struct bio *bio) { - if (fs_bio_integrity_alloc(bio)) + if (fs_bio_integrity_alloc(bio) && + (bio_integrity(bio)->bip_flags & BIP_CHECK_FLAGS)) bio_integrity_generate(bio); } EXPORT_SYMBOL_GPL(fs_bio_integrity_generate); @@ -60,6 +61,9 @@ int fs_bio_integrity_verify(struct bio *bio, sector_t sector, unsigned int size) .bi_size = size, }; + if (!bip || !(bip->bip_flags & BIP_CHECK_FLAGS)) + return 0; + /* * Reinitialize bip->bip_iter. * diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index d9676126c5b5da..52080c7c1a14b5 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -1563,6 +1563,8 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) if (queue_is_mq(q)) memflags = blk_mq_freeze_queue(q); + + mutex_lock(&q->blkcg_mutex); retry: spin_lock_irq(&q->queue_lock); @@ -1572,6 +1574,8 @@ retry: if (blkg->pd[pol->plid]) continue; + if (hlist_unhashed(&blkg->blkcg_node)) + continue; /* If prealloc matches, use it; otherwise try GFP_NOWAIT */ if (blkg == pinned_blkg) { @@ -1625,6 +1629,7 @@ retry: spin_unlock_irq(&q->queue_lock); out: + mutex_unlock(&q->blkcg_mutex); if (queue_is_mq(q)) blk_mq_unfreeze_queue(q, memflags); if (pinned_blkg) diff --git a/block/blk-iocost.c b/block/blk-iocost.c index 8b2aeba2e1e39c..64b92aa3e5d48c 100644 --- a/block/blk-iocost.c +++ b/block/blk-iocost.c @@ -1592,7 +1592,7 @@ static void ioc_lat_stat(struct ioc *ioc, u32 *missed_ppm_ar, u32 *rq_wait_pct_p u64 rq_wait_ns = 0; int cpu, rw; - for_each_online_cpu(cpu) { + for_each_possible_cpu(cpu) { struct ioc_pcpu_stat *stat = per_cpu_ptr(ioc->pcpu_stat, cpu); u64 this_rq_wait_ns; @@ -3063,6 +3063,7 @@ static void iocg_release(struct rcu_head *rcu) static void ioc_pd_free(struct blkg_policy_data *pd) { struct ioc_gq *iocg = pd_to_iocg(pd); + struct blkcg_gq *blkg = pd_to_blkg(pd); struct ioc *ioc = iocg->ioc; unsigned long flags; @@ -3085,6 +3086,12 @@ static void ioc_pd_free(struct blkg_policy_data *pd) hrtimer_cancel(&iocg->waitq_timer); } + /* off ->active_iocgs and timer gone, so nothing can re-arm the delay */ + iocg->delay = 0; + iocg->indelay_since = 0; + if (blkg) + blkcg_clear_delay(blkg); + call_rcu(&pd->rcu_head, iocg_release); } diff --git a/block/blk-iolatency.c b/block/blk-iolatency.c index cef02b6c5fa91d..2caa79a008ad15 100644 --- a/block/blk-iolatency.c +++ b/block/blk-iolatency.c @@ -523,7 +523,7 @@ static void iolatency_check_latencies(struct iolatency_grp *iolat, u64 now) latency_stat_init(iolat, &stat); preempt_disable(); - for_each_online_cpu(cpu) { + for_each_possible_cpu(cpu) { struct latency_stat *s; s = per_cpu_ptr(iolat->stats, cpu); latency_stat_sum(iolat, &stat, s); @@ -925,7 +925,7 @@ static void iolatency_ssd_stat(struct iolatency_grp *iolat, struct seq_file *s) latency_stat_init(iolat, &stat); preempt_disable(); - for_each_online_cpu(cpu) { + for_each_possible_cpu(cpu) { struct latency_stat *s; s = per_cpu_ptr(iolat->stats, cpu); latency_stat_sum(iolat, &stat, s); @@ -1043,6 +1043,15 @@ static void iolat_release(struct rcu_head *rcu) static void iolatency_pd_free(struct blkg_policy_data *pd) { + struct blkcg_gq *blkg = pd_to_blkg(pd); + + /* + * Groups throttled as collateral have min_lat_nsec == 0, so + * iolatency_pd_offline() leaves their delay set. Drop it here, where + * no in-flight bio can re-arm it via check_scale_change(). + */ + if (blkg) + blkcg_clear_delay(blkg); call_rcu(&pd->rcu_head, iolat_release); } diff --git a/block/blk-stat.c b/block/blk-stat.c index de126e1ea5ac42..d57c2fc6bf0642 100644 --- a/block/blk-stat.c +++ b/block/blk-stat.c @@ -83,7 +83,7 @@ static void blk_stat_timer_fn(struct timer_list *t) for (bucket = 0; bucket < cb->buckets; bucket++) blk_rq_stat_init(&cb->stat[bucket]); - for_each_online_cpu(cpu) { + for_each_possible_cpu(cpu) { struct blk_rq_stat *cpu_stat; cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu); diff --git a/block/fops.c b/block/fops.c index a84450d79b3cc9..cb919dea004111 100644 --- a/block/fops.c +++ b/block/fops.c @@ -238,8 +238,10 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, } if (iocb->ki_flags & IOCB_HAS_METADATA) { ret = bio_integrity_map_iter(bio, iocb->private); - if (unlikely(ret)) - goto fail; + if (unlikely(ret)) { + bio_endio_status(bio, errno_to_blk_status(ret)); + break; + } } if (is_read) { diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c index 971818bcdc9dbe..2ee552ab813511 100644 --- a/block/kyber-iosched.c +++ b/block/kyber-iosched.c @@ -275,7 +275,7 @@ static void kyber_timer_fn(struct timer_list *t) bool bad = false; /* Sum all of the per-cpu latency histograms. */ - for_each_online_cpu(cpu) { + for_each_possible_cpu(cpu) { struct kyber_cpu_latency *cpu_latency; cpu_latency = per_cpu_ptr(kqd->cpu_latency, cpu); diff --git a/crypto/acompress.c b/crypto/acompress.c index 032de704eb2ca9..4de1a2ad577f2b 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -559,12 +559,22 @@ EXPORT_SYMBOL_GPL(acomp_walk_virt); struct acomp_req *acomp_request_clone(struct acomp_req *req, size_t total, gfp_t gfp) { + struct crypto_tfm *tfm = req->base.tfm; struct acomp_req *nreq; + size_t len; - nreq = container_of(crypto_request_clone(&req->base, total, gfp), - struct acomp_req, base); - if (nreq == req) + len = sizeof(*req) + + crypto_acomp_reqsize(crypto_acomp_reqtfm(req)); + len = ALIGN(len, CRYPTO_MINALIGN); + + nreq = kzalloc(len, gfp); + if (!nreq) { + req->base.tfm = tfm->fb; return req; + } + + memcpy(nreq, req, sizeof(*req)); + nreq->base.flags &= ~CRYPTO_TFM_REQ_ON_STACK; if (req->src == &req->chain.ssg) nreq->src = &nreq->chain.ssg; diff --git a/crypto/lskcipher.c b/crypto/lskcipher.c index e4328df6e26c05..a79c9a246e5d74 100644 --- a/crypto/lskcipher.c +++ b/crypto/lskcipher.c @@ -95,7 +95,6 @@ static int crypto_lskcipher_crypt_unaligned( while (len >= bs) { unsigned chunk = min((unsigned)PAGE_SIZE, len); - int err; if (chunk > cs) chunk &= ~(cs - 1); diff --git a/drivers/accessibility/speakup/keyhelp.c b/drivers/accessibility/speakup/keyhelp.c index 822ceac8306833..e632c53d6246e1 100644 --- a/drivers/accessibility/speakup/keyhelp.c +++ b/drivers/accessibility/speakup/keyhelp.c @@ -8,6 +8,7 @@ */ #include +#include #include "spk_priv.h" #include "speakup.h" @@ -111,7 +112,7 @@ static void say_key(int key) spk_msg_get(MSG_KEYNAMES_START + (key - 1))); } -static int help_init(void) +static void help_init(void) { char start = SPACE; int i; @@ -120,13 +121,19 @@ static int help_init(void) state_tbl = spk_our_keys[0] + SHIFT_TBL_SIZE + 2; for (i = 0; i < num_funcs; i++) { char *cur_funcname = spk_msg_get(MSG_FUNCNAMES_START + i); + char first_letter; - if (start == *cur_funcname) + first_letter = tolower(*cur_funcname); + + /* Accept only 'a'..'z' to index letter_offsets[] safely */ + if (first_letter < 'a' || first_letter > 'z') + continue; + + if (start == first_letter) continue; - start = *cur_funcname; + start = first_letter; letter_offsets[(start & 31) - 1] = i; } - return 0; } int spk_handle_help(struct vc_data *vc, u_char type, u_char ch, u_short key) @@ -144,7 +151,7 @@ int spk_handle_help(struct vc_data *vc, u_char type, u_char ch, u_short key) synth_printf("%s\n", spk_msg_get(MSG_LEAVING_HELP)); return 1; } - ch |= 32; /* lower case */ + ch = tolower(ch); if (ch < 'a' || ch > 'z') return -1; if (letter_offsets[ch - 'a'] == -1) { diff --git a/drivers/accessibility/speakup/kobjects.c b/drivers/accessibility/speakup/kobjects.c index 0dfdb6608e0222..943ef71b1329bc 100644 --- a/drivers/accessibility/speakup/kobjects.c +++ b/drivers/accessibility/speakup/kobjects.c @@ -92,9 +92,9 @@ static void report_char_chartab_status(int reset, int received, int used, if (reset) { pr_info("%s reset to defaults\n", object_type[do_characters]); } else if (received) { - len = snprintf(buf, sizeof(buf), - " updated %d of %d %s\n", - used, received, object_type[do_characters]); + len = scnprintf(buf, sizeof(buf), + " updated %d of %d %s\n", + used, received, object_type[do_characters]); if (rejected) snprintf(buf + (len - 1), sizeof(buf) - (len - 1), " with %d reject%s\n", diff --git a/drivers/acpi/acpi_platform.c b/drivers/acpi/acpi_platform.c index e3c77a65f61c8d..80ea7121744d0e 100644 --- a/drivers/acpi/acpi_platform.c +++ b/drivers/acpi/acpi_platform.c @@ -108,18 +108,15 @@ static unsigned int acpi_platform_adjust_resources(struct acpi_device *adev, return count; } -static void acpi_platform_fill_resource(struct acpi_device *adev, - const struct resource *src, struct resource *dest) +static void acpi_platform_fill_resource(struct device *parent, + const struct resource *src, + struct resource *dest) { - struct device *parent; - *dest = *src; - /* * If the device has parent we need to take its resources into * account as well because this device might consume part of those. */ - parent = acpi_get_first_physical_node(acpi_dev_parent(adev)); if (parent && dev_is_pci(parent)) dest->parent = pci_find_resource(to_pci_dev(parent), dest); } @@ -147,7 +144,8 @@ static unsigned int acpi_platform_resource_count(struct acpi_resource *ares, voi struct platform_device *acpi_create_platform_device(struct acpi_device *adev, const struct property_entry *properties) { - struct acpi_device *parent = acpi_dev_parent(adev); + struct acpi_device *p = acpi_dev_parent(adev); + struct device *parent __free(put_device) = acpi_bus_get_primary_device(p); struct platform_device *pdev = NULL; struct platform_device_info pdevinfo; const struct acpi_device_id *match; @@ -193,7 +191,7 @@ struct platform_device *acpi_create_platform_device(struct acpi_device *adev, rentry->res, resources, count); - acpi_platform_fill_resource(adev, rentry->res, + acpi_platform_fill_resource(parent, rentry->res, &resources[count++]); } acpi_dev_free_resource_list(&resource_list); @@ -206,7 +204,7 @@ struct platform_device *acpi_create_platform_device(struct acpi_device *adev, * attached to it, that physical device should be the parent of the * platform device we are about to create. */ - pdevinfo.parent = parent ? acpi_get_first_physical_node(parent) : NULL; + pdevinfo.parent = parent; pdevinfo.name = dev_name(&adev->dev); pdevinfo.id = PLATFORM_DEVID_NONE; pdevinfo.res = resources; diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c index f8fa0d9a1f4c79..365626827d61ce 100644 --- a/drivers/acpi/battery.c +++ b/drivers/acpi/battery.c @@ -153,27 +153,28 @@ static int acpi_battery_technology(struct acpi_battery *battery) static int acpi_battery_get_state(struct acpi_battery *battery); -static int acpi_battery_is_charged(struct acpi_battery *battery) +static bool acpi_battery_is_full(struct acpi_battery *battery) { - /* charging, discharging, critical low or charge limited */ - if (battery->state != 0) - return 0; - /* battery not reporting charge */ if (battery->capacity_now == ACPI_BATTERY_VALUE_UNKNOWN || battery->capacity_now == 0) - return 0; + return false; /* good batteries update full_charge as the batteries degrade */ if (battery->full_charge_capacity == battery->capacity_now) - return 1; + return true; /* fallback to using design values for broken batteries */ - if (battery->design_capacity <= battery->capacity_now) - return 1; + return battery->design_capacity <= battery->capacity_now; +} - /* we don't do any sort of metric based on percentages */ - return 0; +static int acpi_battery_is_charged(struct acpi_battery *battery) +{ + /* charging, discharging, critical low or charge limited */ + if (battery->state != 0) + return 0; + + return acpi_battery_is_full(battery); } static bool acpi_battery_is_degraded(struct acpi_battery *battery) @@ -226,13 +227,14 @@ static int acpi_battery_get_property(struct power_supply *psy, return 0; } else if (battery->state & ACPI_BATTERY_STATE_CHARGING) - /* Validate the status by checking the current. */ - if (battery->rate_now != ACPI_BATTERY_VALUE_UNKNOWN && - battery->rate_now == 0) { - /* On charge but no current (0W/0mA). */ - val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; - } else { + /* Check the rate and capacity to validate the status. */ + if (!acpi_battery_is_full(battery) || + (battery->rate_now != ACPI_BATTERY_VALUE_UNKNOWN && + battery->rate_now > 0)) { val->intval = POWER_SUPPLY_STATUS_CHARGING; + } else { + /* Full and zero rate. */ + val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; } else if (battery->state & ACPI_BATTERY_STATE_CHARGE_LIMITING) val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c index a30a904f6535f4..7a2458b00a6613 100644 --- a/drivers/acpi/bus.c +++ b/drivers/acpi/bus.c @@ -335,7 +335,7 @@ static int acpi_osc_handshake(acpi_handle handle, const char *uuid_str, .length = bufsize * sizeof(u32), }; struct acpi_buffer output; - u32 *retbuf, test; + u32 *retbuf, test, errors; guid_t guid; int ret, i; @@ -395,10 +395,18 @@ static int acpi_osc_handshake(acpi_handle handle, const char *uuid_str, * Clear the feature bits in capbuf[] that have not been acknowledged. * After that, capbuf[] contains the resultant feature mask. */ - for (i = OSC_QUERY_DWORD + 1; i < bufsize; i++) + for (i = OSC_QUERY_DWORD + 1, test = 0; i < bufsize; i++) { + test |= capbuf[i] & ~retbuf[i]; capbuf[i] &= retbuf[i]; + } - if (retbuf[OSC_QUERY_DWORD] & OSC_ERROR_MASK) { + errors = retbuf[OSC_QUERY_DWORD] & OSC_ERROR_MASK; + /* + * Some platforms set OSC_CAPABILITIES_MASK_ERROR even though they + * acknowledge all of the requested features, so avoid complaining in + * those cases unless any other error bits are also set. + */ + if (errors && (test || errors != OSC_CAPABILITIES_MASK_ERROR)) { /* * Complain about the unexpected errors and print diagnostic * information related to them. @@ -801,30 +809,52 @@ static int __init acpi_setup_sb_notify_handler(void) Device Matching -------------------------------------------------------------------------- */ + +static struct device *primary_physical_device(struct acpi_device *adev) +{ + struct acpi_device_physical_node *pn; + + pn = list_first_entry_or_null(&adev->physical_node_list, + struct acpi_device_physical_node, node); + if (pn) + return pn->dev; + + return NULL; +} + +/** + * acpi_bus_get_primary_device - Get first physical device for a given ACPI one + * @adev: ACPI device to get the first physical device for. + * + * Find the first physical device for which @adev is the ACPI companion and + * reference count it if present. + * + * Return: Pointer to the first physical counterpart of @adev or NULL if there + * are none. Callers are responsible for invoking put_device() on the returned + * device. + */ +struct device *acpi_bus_get_primary_device(struct acpi_device *adev) +{ + if (!adev) + return NULL; + + guard(mutex)(&adev->physical_node_lock); + + return get_device(primary_physical_device(adev)); +} +EXPORT_SYMBOL_GPL(acpi_bus_get_primary_device); + /** - * acpi_get_first_physical_node - Get first physical node of an ACPI device + * acpi_get_first_physical_node - Find first physical node of an ACPI device * @adev: ACPI device in question * * Return: First physical node of ACPI device @adev */ struct device *acpi_get_first_physical_node(struct acpi_device *adev) { - struct mutex *physical_node_lock = &adev->physical_node_lock; - struct device *phys_dev; - - mutex_lock(physical_node_lock); - if (list_empty(&adev->physical_node_list)) { - phys_dev = NULL; - } else { - const struct acpi_device_physical_node *node; - - node = list_first_entry(&adev->physical_node_list, - struct acpi_device_physical_node, node); + guard(mutex)(&adev->physical_node_lock); - phys_dev = node->dev; - } - mutex_unlock(physical_node_lock); - return phys_dev; + return primary_physical_device(adev); } EXPORT_SYMBOL_GPL(acpi_get_first_physical_node); diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c index 64ad4cfa6208bd..a89f10256dbb85 100644 --- a/drivers/acpi/ec.c +++ b/drivers/acpi/ec.c @@ -1510,6 +1510,24 @@ static bool install_gpio_irq_event_handler(struct acpi_ec *ec) IRQF_SHARED | IRQF_ONESHOT, "ACPI EC", ec) >= 0; } +static int ec_prepare_gpio_irq(struct acpi_ec *ec, struct acpi_device *device) +{ + int irq; + + if (!device || ec->gpe >= 0 || ec->irq >= 0) + return 0; + + /* ACPI reduced hardware platforms use a GpioInt from _CRS. */ + irq = acpi_dev_gpio_irq_get(device, 0); + if (irq == -EPROBE_DEFER) + return irq; + + if (irq >= 0) + ec->irq = irq; + + return 0; +} + /** * ec_install_handlers - Install service callbacks and register query methods. * @ec: Target EC. @@ -1524,7 +1542,6 @@ static bool install_gpio_irq_event_handler(struct acpi_ec *ec) * Return: * -ENODEV if the address space handler cannot be installed, which means * "unable to handle transactions", - * -EPROBE_DEFER if GPIO IRQ acquisition needs to be deferred, * or 0 (success) otherwise. */ static int ec_install_handlers(struct acpi_ec *ec, struct acpi_device *device, @@ -1557,19 +1574,6 @@ static int ec_install_handlers(struct acpi_ec *ec, struct acpi_device *device, if (!device) return 0; - if (ec->gpe < 0) { - /* ACPI reduced hardware platforms use a GpioInt from _CRS. */ - int irq = acpi_dev_gpio_irq_get(device, 0); - /* - * Bail out right away for deferred probing or complete the - * initialization regardless of any other errors. - */ - if (irq == -EPROBE_DEFER) - return -EPROBE_DEFER; - else if (irq >= 0) - ec->irq = irq; - } - if (!test_bit(EC_FLAGS_QUERY_METHODS_INSTALLED, &ec->flags)) { /* Find and register all query methods */ acpi_walk_namespace(ACPI_TYPE_METHOD, ec->handle, 1, @@ -1647,6 +1651,14 @@ static int acpi_ec_setup(struct acpi_ec *ec, struct acpi_device *device, bool ca { int ret; + /* + * GPIO IRQ lookup can defer. Do it before publishing the EC + * OpRegion to AML to avoid a spurious _REG(disconnect). + */ + ret = ec_prepare_gpio_irq(ec, device); + if (ret) + return ret; + /* First EC capable of handling transactions */ if (!first_ec) first_ec = ec; diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c index 4c06c3ffd0cbb5..d836f46319f266 100644 --- a/drivers/acpi/pci_root.c +++ b/drivers/acpi/pci_root.c @@ -574,6 +574,13 @@ static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm) return; } + if (!is_pcie(root) && !is_cxl(root) && !acpi_has_method(handle, "_OSC")) { + dev_dbg(&device->dev, "Non-PCIe host bridge without _OSC, skipping\n"); + + *no_aspm = 1; + return; + } + support = calculate_support(); decode_osc_support(root, "OS supports", support); @@ -615,10 +622,6 @@ static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm) */ *no_aspm = 1; - /* _OSC is optional for PCI host bridges */ - if (status == AE_NOT_FOUND && !is_pcie(root)) - return; - if (control) { decode_osc_control(root, "OS requested", requested); decode_osc_control(root, "platform willing to grant", control); @@ -725,7 +728,6 @@ static int acpi_pci_root_add(struct acpi_device *device, dev_err(&device->dev, "Bus %04x:%02x not present in PCI namespace\n", root->segment, (unsigned int)root->secondary.start); - device->driver_data = NULL; result = -ENODEV; goto remove_dmar; } @@ -765,6 +767,7 @@ remove_dmar: if (hotadd) dmar_device_remove(handle); end: + device->driver_data = NULL; kfree(root); return result; } @@ -788,6 +791,7 @@ static void acpi_pci_root_remove(struct acpi_device *device) pci_unlock_rescan_remove(); + device->driver_data = NULL; kfree(root); } diff --git a/drivers/acpi/processor_core.c b/drivers/acpi/processor_core.c index a4498357bd165a..3bf076c150fa1d 100644 --- a/drivers/acpi/processor_core.c +++ b/drivers/acpi/processor_core.c @@ -336,11 +336,26 @@ int acpi_get_cpuid(acpi_handle handle, int type, u32 acpi_id) EXPORT_SYMBOL_GPL(acpi_get_cpuid); #ifdef CONFIG_ACPI_HOTPLUG_IOAPIC -static int get_ioapic_id(struct acpi_subtable_header *entry, u32 gsi_base, +static bool madt_entry_is_valid(struct acpi_subtable_header *entry, + unsigned long end) +{ + unsigned long start = (unsigned long)entry; + + if (start >= end || end - start < sizeof(*entry)) + return false; + + return entry->length >= sizeof(*entry) && entry->length <= end - start; +} + +static int get_ioapic_id(struct acpi_subtable_header *entry, + const unsigned long end, u32 gsi_base, u64 *phys_addr, int *ioapic_id) { struct acpi_madt_io_apic *ioapic = (struct acpi_madt_io_apic *)entry; + if (!madt_entry_is_valid(entry, end) || BAD_MADT_ENTRY(ioapic, end)) + return 0; + if (ioapic->global_irq_base != gsi_base) return 0; @@ -361,17 +376,19 @@ static int parse_madt_ioapic_entry(u32 gsi_base, u64 *phys_addr) return apic_id; entry = (unsigned long)madt; + if (madt->header.length < sizeof(*madt)) + return apic_id; madt_end = entry + madt->header.length; /* Parse all entries looking for a match. */ entry += sizeof(struct acpi_table_madt); - while (entry + sizeof(struct acpi_subtable_header) < madt_end) { + while (madt_entry_is_valid((struct acpi_subtable_header *)entry, + madt_end)) { hdr = (struct acpi_subtable_header *)entry; if (hdr->type == ACPI_MADT_TYPE_IO_APIC && - get_ioapic_id(hdr, gsi_base, phys_addr, &apic_id)) + get_ioapic_id(hdr, madt_end, gsi_base, phys_addr, &apic_id)) break; - else - entry += hdr->length; + entry += hdr->length; } return apic_id; @@ -398,7 +415,9 @@ static int parse_mat_ioapic_entry(acpi_handle handle, u32 gsi_base, header = (struct acpi_subtable_header *)obj->buffer.pointer; if (header->type == ACPI_MADT_TYPE_IO_APIC) - get_ioapic_id(header, gsi_base, phys_addr, &apic_id); + get_ioapic_id(header, + (unsigned long)header + obj->buffer.length, + gsi_base, phys_addr, &apic_id); exit: kfree(buffer.pointer); diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c index cda8fd7200004f..cdc2ae1632b21a 100644 --- a/drivers/acpi/processor_driver.c +++ b/drivers/acpi/processor_driver.c @@ -285,6 +285,12 @@ err: unregister_idle_drv: acpi_processor_unregister_idle_driver(); + if (acpi_processor_cpufreq_init) { + cpufreq_unregister_notifier(&acpi_processor_notifier_block, + CPUFREQ_POLICY_NOTIFIER); + acpi_processor_cpufreq_init = false; + } + return result; } diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index 4482cf28f56aee..d573f201295ae6 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -927,6 +927,13 @@ static int acpi_processor_evaluate_lpi(acpi_handle handle, if (obj->type == ACPI_TYPE_BUFFER) { struct acpi_power_register *reg; + if (obj->buffer.length < sizeof(*reg)) { + acpi_handle_debug(handle, + "Invalid register data for _LPI state %d\n", + state_idx); + continue; + } + reg = (struct acpi_power_register *)obj->buffer.pointer; if (reg->space_id != ACPI_ADR_SPACE_SYSTEM_IO && reg->space_id != ACPI_ADR_SPACE_FIXED_HARDWARE) @@ -945,13 +952,6 @@ static int acpi_processor_evaluate_lpi(acpi_handle handle, continue; } - /* elements[7,8] skipped for now i.e. Residency/Usage counter*/ - - obj = pkg_elem + 9; - if (obj->type == ACPI_TYPE_STRING) - strscpy(lpi_state->desc, obj->string.pointer, - ACPI_CX_DESC_LEN); - lpi_state->index = state_idx; if (obj_get_integer(pkg_elem + 0, &lpi_state->min_residency)) { pr_debug("No min. residency found, assuming 10 us\n"); @@ -974,6 +974,20 @@ static int acpi_processor_evaluate_lpi(acpi_handle handle, if (obj_get_integer(pkg_elem + 5, &lpi_state->enable_parent_state)) lpi_state->enable_parent_state = 0; + + /* Skip elements [7-8] i.e. Residency/Usage counters. */ + + /* + * Avoid out-of-bounds access if the size of the package is less + * than expected. + */ + if (element->package.count < 10) + continue; + + obj = pkg_elem + 9; + if (obj->type == ACPI_TYPE_STRING) + strscpy(lpi_state->desc, obj->string.pointer, + ACPI_CX_DESC_LEN); } acpi_handle_debug(handle, "Found %d power states\n", state_idx); diff --git a/drivers/acpi/riscv/irq.c b/drivers/acpi/riscv/irq.c index 9b88d0993e8807..0cdec5dd575ebf 100644 --- a/drivers/acpi/riscv/irq.c +++ b/drivers/acpi/riscv/irq.c @@ -299,6 +299,7 @@ static acpi_status riscv_acpi_irq_get_parent(struct acpi_resource *ares, void *c return AE_OK; ctx->handle = riscv_acpi_get_gsi_handle(eirq->interrupts[ctx->index]); + ctx->rc = 0; return AE_CTRL_TERMINATE; } @@ -314,10 +315,22 @@ static int riscv_acpi_irq_get_dep(acpi_handle handle, unsigned int index, acpi_h acpi_walk_resources(handle, METHOD_NAME__CRS, riscv_acpi_irq_get_parent, &ctx); *gsi_handle = ctx.handle; - if (*gsi_handle) - return 1; - return 0; + return ctx.rc; +} + +static bool acpi_prt_entry_valid(void *prt_entry) +{ + struct acpi_pci_routing_table *entry = prt_entry; + + return entry && entry->length > 0; +} + +static void *acpi_prt_next_entry(void *prt_entry) +{ + struct acpi_pci_routing_table *entry = prt_entry; + + return prt_entry + entry->length; } static u32 riscv_acpi_add_prt_dep(acpi_handle handle) @@ -338,9 +351,11 @@ static u32 riscv_acpi_add_prt_dep(acpi_handle handle) } entry = buffer.pointer; - while (entry && (entry->length > 0)) { + for (; acpi_prt_entry_valid(entry); entry = acpi_prt_next_entry(entry)) { if (entry->source[0]) { - acpi_get_handle(handle, entry->source, &link_handle); + status = acpi_get_handle(handle, entry->source, &link_handle); + if (ACPI_FAILURE(status)) + continue; dep_devices.count = 1; dep_devices.handles = kzalloc_objs(*dep_devices.handles, 1); @@ -364,9 +379,6 @@ static u32 riscv_acpi_add_prt_dep(acpi_handle handle) dep_devices.handles[0] = gsi_handle; count += acpi_scan_add_dep(handle, &dep_devices); } - - entry = (struct acpi_pci_routing_table *) - ((unsigned long)entry + entry->length); } kfree(buffer.pointer); @@ -381,8 +393,11 @@ static u32 riscv_acpi_add_irq_dep(acpi_handle handle) int i; for (i = 0; - riscv_acpi_irq_get_dep(handle, i, &gsi_handle); + !riscv_acpi_irq_get_dep(handle, i, &gsi_handle); i++) { + if (!gsi_handle) + continue; + dep_devices.count = 1; dep_devices.handles = kzalloc_objs(*dep_devices.handles, 1); if (!dep_devices.handles) { diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index 9a7ac2eb9ce0b8..99af95f3386525 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -524,12 +524,10 @@ static void acpi_device_release(struct device *dev) kfree(acpi_dev); } -static void acpi_device_del(struct acpi_device *device) +static void acpi_device_cleanup(struct acpi_device *device) { struct acpi_device_bus_id *acpi_device_bus_id; - mutex_lock(&acpi_device_lock); - list_for_each_entry(acpi_device_bus_id, &acpi_bus_id_list, node) if (!strcmp(acpi_device_bus_id->bus_id, acpi_device_hid(device))) { @@ -544,6 +542,13 @@ static void acpi_device_del(struct acpi_device *device) } list_del(&device->wakeup_list); +} + +static void acpi_device_del(struct acpi_device *device) +{ + mutex_lock(&acpi_device_lock); + + acpi_device_cleanup(device); mutex_unlock(&acpi_device_lock); @@ -803,7 +808,7 @@ int acpi_device_add(struct acpi_device *device) err: mutex_lock(&acpi_device_lock); - list_del(&device->wakeup_list); + acpi_device_cleanup(device); err_unlock: mutex_unlock(&acpi_device_lock); @@ -2208,29 +2213,27 @@ static void acpi_create_video_bus_device(struct acpi_device *adev, struct auxiliary_device *aux_dev; static unsigned int aux_dev_id; + struct device *phys_parent __free(put_device) = acpi_bus_get_primary_device(parent); + if (!phys_parent) + return; + aux_dev = kzalloc_obj(*aux_dev); if (!aux_dev) return; aux_dev->id = aux_dev_id++; aux_dev->name = "video_bus"; - aux_dev->dev.parent = acpi_get_first_physical_node(parent); - if (!aux_dev->dev.parent) - goto err; - + aux_dev->dev.parent = phys_parent; aux_dev->dev.release = acpi_video_bus_device_release; - if (auxiliary_device_init(aux_dev)) - goto err; + if (auxiliary_device_init(aux_dev)) { + kfree(aux_dev); + return; + } ACPI_COMPANION_SET(&aux_dev->dev, adev); if (__auxiliary_device_add(aux_dev, "acpi")) auxiliary_device_uninit(aux_dev); - - return; - -err: - kfree(aux_dev); } struct acpi_scan_system_dev { diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c index 458efa4fe9d4b0..4c1c3b0b5499db 100644 --- a/drivers/acpi/video_detect.c +++ b/drivers/acpi/video_detect.c @@ -137,8 +137,10 @@ static int video_detect_portege_r100(const struct dmi_system_id *d) struct pci_dev *dev; /* Search for Trident CyberBlade XP4m32 to confirm Portégé R100 */ dev = pci_get_device(PCI_VENDOR_ID_TRIDENT, 0x2100, NULL); - if (dev) + if (dev) { acpi_backlight_dmi = acpi_backlight_vendor; + pci_dev_put(dev); + } return 0; } diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c index 70701d3bc81c6f..9f9c72727a0590 100644 --- a/drivers/base/cacheinfo.c +++ b/drivers/base/cacheinfo.c @@ -401,9 +401,14 @@ static int cache_setup_properties(unsigned int cpu) else if (!acpi_disabled) ret = cache_setup_acpi(cpu); - // Assume there is no cache information available in DT/ACPI from now. - if (ret && use_arch_cache_info()) + /* + * No DT/ACPI cache nodes; fall back to arch-derived topology (e.g. + * arm64 CLIDR_EL1) and clear the error to avoid a spurious warning. + */ + if (ret && use_arch_cache_info()) { use_arch_info = true; + ret = 0; + } return ret; } diff --git a/drivers/base/firmware_loader/fallback.c b/drivers/base/firmware_loader/fallback.c index 3ef0b312ae7190..00d6b6e594616a 100644 --- a/drivers/base/firmware_loader/fallback.c +++ b/drivers/base/firmware_loader/fallback.c @@ -95,6 +95,16 @@ static int fw_load_sysfs_fallback(struct fw_sysfs *fw_sysfs, long timeout) retval = -EINTR; goto out; } + + /* + * device_add() exposes the loading interface before pending_list is + * linked into pending_fw_head, so fw_state_done() may run first. + */ + if (fw_state_is_done(fw_priv)) { + mutex_unlock(&fw_lock); + goto out; + } + list_add(&fw_priv->pending_list, &pending_fw_head); mutex_unlock(&fw_lock); diff --git a/drivers/base/soc.c b/drivers/base/soc.c index 65ce72d4923036..af7d71393774bf 100644 --- a/drivers/base/soc.c +++ b/drivers/base/soc.c @@ -191,6 +191,7 @@ EXPORT_SYMBOL_GPL(soc_device_unregister); static int __init soc_bus_register(void) { + struct soc_device *soc_dev; int ret; ret = bus_register(&soc_bus_type); @@ -198,10 +199,20 @@ static int __init soc_bus_register(void) return ret; soc_bus_registered = true; - if (early_soc_dev_attr) - return PTR_ERR(soc_device_register(early_soc_dev_attr)); + if (early_soc_dev_attr) { + soc_dev = soc_device_register(early_soc_dev_attr); + if (IS_ERR(soc_dev)) { + ret = PTR_ERR(soc_dev); + goto err_unregister_bus; + } + } return 0; + +err_unregister_bus: + soc_bus_registered = false; + bus_unregister(&soc_bus_type); + return ret; } core_initcall(soc_bus_register); diff --git a/drivers/base/swnode.c b/drivers/base/swnode.c index 869228a65cb365..2bc76f01eb77dd 100644 --- a/drivers/base/swnode.c +++ b/drivers/base/swnode.c @@ -537,7 +537,7 @@ software_node_get_reference_args(const struct fwnode_handle *fwnode, if (prop->is_inline) return -EINVAL; - if ((index + 1) * sizeof(*ref) > prop->length) + if (index >= prop->length / sizeof(*ref)) return -ENOENT; ref_array = prop->pointer; diff --git a/drivers/base/transport_class.c b/drivers/base/transport_class.c index 416e9f819df51f..351c3d3ce6a080 100644 --- a/drivers/base/transport_class.c +++ b/drivers/base/transport_class.c @@ -168,11 +168,14 @@ static int transport_add_class_device(struct attribute_container *cont, if (tcont->encryption) { error = sysfs_create_group(&classdev->kobj, tcont->encryption); if (error) - goto err_del; + goto err_del_statistics; } return 0; +err_del_statistics: + if (tcont->statistics) + sysfs_remove_group(&classdev->kobj, tcont->statistics); err_del: attribute_container_class_device_del(classdev); err_remove: diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index f214a616386c8e..113bdb868c469f 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -3048,6 +3048,8 @@ static int mtip_block_ioctl(struct block_device *dev, if (!dd) return -ENOTTY; + guard(mutex)(&dd->ioctl_mutex); + if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) return -ENOTTY; @@ -3086,6 +3088,8 @@ static int mtip_block_compat_ioctl(struct block_device *dev, if (!dd) return -ENOTTY; + guard(mutex)(&dd->ioctl_mutex); + if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) return -ENOTTY; @@ -3721,6 +3725,7 @@ static int mtip_pci_probe(struct pci_dev *pdev, dd = kzalloc_node(sizeof(struct driver_data), GFP_KERNEL, my_node); if (!dd) return -ENOMEM; + mutex_init(&dd->ioctl_mutex); /* Attach the private data to this PCI device. */ pci_set_drvdata(pdev, dd); @@ -3887,6 +3892,7 @@ static void mtip_pci_remove(struct pci_dev *pdev) } set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag); + mutex_lock(&dd->ioctl_mutex); if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) del_gendisk(dd->disk); @@ -3915,6 +3921,7 @@ static void mtip_pci_remove(struct pci_dev *pdev) /* De-initialize the protocol layer. */ mtip_hw_exit(dd); + mutex_unlock(&dd->ioctl_mutex); if (dd->isr_workq) { destroy_workqueue(dd->isr_workq); diff --git a/drivers/block/mtip32xx/mtip32xx.h b/drivers/block/mtip32xx/mtip32xx.h index f7328f19ac5c27..0963c07b58457b 100644 --- a/drivers/block/mtip32xx/mtip32xx.h +++ b/drivers/block/mtip32xx/mtip32xx.h @@ -12,6 +12,7 @@ #define __MTIP32XX_H__ #include +#include #include #include #include @@ -432,6 +433,7 @@ struct driver_data { struct request_queue *queue; /* Our request queue. */ struct blk_mq_tag_set tags; /* blk_mq tags */ + struct mutex ioctl_mutex; struct mtip_port *port; /* Pointer to the port data structure. */ diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index f8c0fd57e041a8..4a9beec211b926 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -66,7 +66,7 @@ struct nullb_page { #define NULLB_PAGE_FREE (MAP_SZ - 2) static LIST_HEAD(nullb_list); -static struct mutex lock; +static DEFINE_MUTEX(lock); static int null_major; static DEFINE_IDA(nullb_indexes); static struct blk_mq_tag_set tag_set; @@ -340,7 +340,15 @@ static ssize_t nullb_device_bool_attr_store(bool *val, const char *page, return count; } -/* The following macro should only be used with TYPE = {uint, ulong, bool}. */ +/* + * The following macro should only be used with TYPE = {uint, ulong, bool}. + * + * The device configuration is modified under the global lock to serialize + * attribute changes against null_add_dev() and null_del_dev(): without this, + * an attribute could be changed while null_add_dev() is running, that is, + * before NULLB_DEV_FL_CONFIGURED is set, which would let null_add_dev() + * observe inconsistent values for the device configuration. + */ #define NULLB_DEVICE_ATTR(NAME, TYPE, APPLY) \ static ssize_t \ nullb_device_##NAME##_show(struct config_item *item, char *page) \ @@ -360,6 +368,7 @@ nullb_device_##NAME##_store(struct config_item *item, const char *page, \ ret = nullb_device_##TYPE##_attr_store(&new_value, page, count);\ if (ret < 0) \ return ret; \ + guard(mutex)(&lock); \ if (apply_fn) \ ret = apply_fn(dev, new_value); \ else if (test_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags)) \ @@ -379,9 +388,20 @@ static int nullb_update_nr_hw_queues(struct nullb_device *dev, struct blk_mq_tag_set *set; int ret, nr_hw_queues; + lockdep_assert_held(&lock); + if (!dev->nullb) return 0; + /* + * A shared tag_set is mapped via the module-wide queue counts, so a + * per-device resize is meaningless. On shrink it would also leave + * mq_map[] pointing at NULLed hctx slots, causing a NULL deref in + * blk_mq_map_swqueue(). Reject it. + */ + if (dev->shared_tags) + return -EINVAL; + /* * Make sure at least one submit queue exists. */ @@ -421,25 +441,13 @@ static int nullb_update_nr_hw_queues(struct nullb_device *dev, static int nullb_apply_submit_queues(struct nullb_device *dev, unsigned int submit_queues) { - int ret; - - mutex_lock(&lock); - ret = nullb_update_nr_hw_queues(dev, submit_queues, dev->poll_queues); - mutex_unlock(&lock); - - return ret; + return nullb_update_nr_hw_queues(dev, submit_queues, dev->poll_queues); } static int nullb_apply_poll_queues(struct nullb_device *dev, unsigned int poll_queues) { - int ret; - - mutex_lock(&lock); - ret = nullb_update_nr_hw_queues(dev, dev->submit_queues, poll_queues); - mutex_unlock(&lock); - - return ret; + return nullb_update_nr_hw_queues(dev, dev->submit_queues, poll_queues); } NULLB_DEVICE_ATTR(size, ulong, NULL); @@ -579,8 +587,13 @@ static ssize_t nullb_device_zone_readonly_store(struct config_item *item, const char *page, size_t count) { struct nullb_device *dev = to_nullb_device(item); + ssize_t ret; - return zone_cond_store(dev, page, count, BLK_ZONE_COND_READONLY); + mutex_lock(&lock); + ret = zone_cond_store(dev, page, count, BLK_ZONE_COND_READONLY); + mutex_unlock(&lock); + + return ret; } CONFIGFS_ATTR_WO(nullb_device_, zone_readonly); @@ -588,8 +601,13 @@ static ssize_t nullb_device_zone_offline_store(struct config_item *item, const char *page, size_t count) { struct nullb_device *dev = to_nullb_device(item); + ssize_t ret; + + mutex_lock(&lock); + ret = zone_cond_store(dev, page, count, BLK_ZONE_COND_OFFLINE); + mutex_unlock(&lock); - return zone_cond_store(dev, page, count, BLK_ZONE_COND_OFFLINE); + return ret; } CONFIGFS_ATTR_WO(nullb_device_, zone_offline); @@ -836,7 +854,6 @@ static void null_free_dev(struct nullb_device *dev) if (!dev) return; - null_free_zoned_dev(dev); badblocks_exit(&dev->badblocks); kfree(dev); } @@ -1777,6 +1794,7 @@ static void null_del_dev(struct nullb *nullb) } put_disk(nullb->disk); + null_free_zoned_dev(dev); if (nullb->tag_set == &nullb->__tag_set) blk_mq_free_tag_set(nullb->tag_set); kfree(nullb->queues); @@ -2162,17 +2180,9 @@ static int __init null_init(void) config_group_init(&nullb_subsys.su_group); mutex_init(&nullb_subsys.su_mutex); - ret = configfs_register_subsystem(&nullb_subsys); - if (ret) - return ret; - - mutex_init(&lock); - null_major = register_blkdev(0, "nullb"); - if (null_major < 0) { - ret = null_major; - goto err_conf; - } + if (null_major < 0) + return null_major; for (i = 0; i < nr_devices; i++) { ret = null_create_dev(); @@ -2180,6 +2190,10 @@ static int __init null_init(void) goto err_dev; } + ret = configfs_register_subsystem(&nullb_subsys); + if (ret) + goto err_dev; + pr_info("module loaded\n"); return 0; @@ -2189,8 +2203,8 @@ err_dev: null_destroy_dev(nullb); } unregister_blkdev(null_major, "nullb"); -err_conf: - configfs_unregister_subsystem(&nullb_subsys); + if (tag_set.ops) + blk_mq_free_tag_set(&tag_set); return ret; } @@ -2211,8 +2225,6 @@ static void __exit null_exit(void) if (tag_set.ops) blk_mq_free_tag_set(&tag_set); - - mutex_destroy(&lock); } module_init(null_init); diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 037e08cb97ce04..b71283588ea772 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -970,7 +970,7 @@ static int ublk_validate_params(const struct ublk_device *ub) if (p->max_sectors < PAGE_SECTORS) return -EINVAL; - if (ublk_dev_is_zoned(ub) && !p->chunk_sectors) + if (ublk_dev_is_zoned(ub) && !is_power_of_2(p->chunk_sectors)) return -EINVAL; } else return -EINVAL; @@ -1475,7 +1475,10 @@ static unsigned int ublk_map_io(const struct ublk_queue *ubq, struct iov_iter iter; const int dir = ITER_DEST; - import_ubuf(dir, u64_to_user_ptr(io->buf.addr), rq_bytes, &iter); + if (import_ubuf(dir, u64_to_user_ptr(io->buf.addr), rq_bytes, + &iter) < 0) + return 0; + return ublk_copy_user_pages(req, 0, &iter, dir); } return rq_bytes; @@ -1496,7 +1499,10 @@ static unsigned int ublk_unmap_io(bool need_map, WARN_ON_ONCE(io->res > rq_bytes); - import_ubuf(dir, u64_to_user_ptr(io->buf.addr), io->res, &iter); + if (import_ubuf(dir, u64_to_user_ptr(io->buf.addr), io->res, + &iter) < 0) + return 0; + return ublk_copy_user_pages(req, 0, &iter, dir); } return rq_bytes; @@ -1584,8 +1590,14 @@ static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io, * * Re-read simply for this unlikely case. */ - if (unlikely(unmapped_bytes < io->res)) + if (unlikely(unmapped_bytes < io->res)) { + if (unlikely(!unmapped_bytes)) { + res = BLK_STS_IOERR; + goto exit; + } + io->res = unmapped_bytes; + } /* * Run bio->bi_end_io() with softirqs disabled. If the final fput @@ -3081,18 +3093,19 @@ static inline int ublk_check_cmd_op(u32 cmd_op) return 0; } -static inline int ublk_set_auto_buf_reg(struct ublk_io *io, struct io_uring_cmd *cmd) +/* Must run before ublk_fill_io_cmd() / __ublk_fetch(). */ +static inline int ublk_validate_io_buf(const struct ublk_device *ub, + struct io_uring_cmd *cmd, + struct ublk_auto_buf_reg *buf) { - struct ublk_auto_buf_reg buf; - - buf = ublk_sqe_addr_to_auto_buf_reg(READ_ONCE(cmd->sqe->addr)); + if (!ublk_dev_support_auto_buf_reg(ub)) + return 0; - if (buf.reserved0 || buf.reserved1) + *buf = ublk_sqe_addr_to_auto_buf_reg(READ_ONCE(cmd->sqe->addr)); + if (buf->reserved0 || buf->reserved1) return -EINVAL; - - if (buf.flags & ~UBLK_AUTO_BUF_REG_F_MASK) + if (buf->flags & ~UBLK_AUTO_BUF_REG_F_MASK) return -EINVAL; - io->buf.auto_reg = buf; return 0; } @@ -3113,17 +3126,25 @@ static void ublk_clear_auto_buf_reg(struct ublk_io *io, * responsibility for unregistering the buffer, otherwise * this ublk request gets stuck. */ - if (io->buf_ctx_handle == io_uring_cmd_ctx_handle(cmd)) + if (buf_idx && + io->buf_ctx_handle == io_uring_cmd_ctx_handle(cmd)) *buf_idx = io->buf.auto_reg.index; } } -static int ublk_handle_auto_buf_reg(struct ublk_io *io, - struct io_uring_cmd *cmd, - u16 *buf_idx) +static inline void ublk_apply_io_buf(const struct ublk_device *ub, + struct ublk_io *io, + struct io_uring_cmd *cmd, + unsigned long buf_addr, + const struct ublk_auto_buf_reg *auto_buf, + u16 *buf_idx) { - ublk_clear_auto_buf_reg(io, cmd, buf_idx); - return ublk_set_auto_buf_reg(io, cmd); + if (ublk_dev_support_auto_buf_reg(ub)) { + ublk_clear_auto_buf_reg(io, cmd, buf_idx); + io->buf.auto_reg = *auto_buf; + } else { + io->buf.addr = buf_addr; + } } /* Once we return, `io->req` can't be used any more */ @@ -3140,18 +3161,6 @@ ublk_fill_io_cmd(struct ublk_io *io, struct io_uring_cmd *cmd) return req; } -static inline int -ublk_config_io_buf(const struct ublk_device *ub, struct ublk_io *io, - struct io_uring_cmd *cmd, unsigned long buf_addr, - u16 *buf_idx) -{ - if (ublk_dev_support_auto_buf_reg(ub)) - return ublk_handle_auto_buf_reg(io, cmd, buf_idx); - - io->buf.addr = buf_addr; - return 0; -} - static inline void ublk_prep_cancel(struct io_uring_cmd *cmd, unsigned int issue_flags, struct ublk_queue *ubq, unsigned int tag) @@ -3292,6 +3301,7 @@ static int __ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub, static int ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub, struct ublk_io *io, __u64 buf_addr, u16 q_id) { + struct ublk_auto_buf_reg auto_buf; int ret; /* @@ -3300,11 +3310,13 @@ static int ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub, * FETCH, so it is fine even for IO_URING_F_NONBLOCK. */ mutex_lock(&ub->mutex); - ret = __ublk_fetch(cmd, ub, io, q_id); - if (!ret) - ret = ublk_config_io_buf(ub, io, cmd, buf_addr, NULL); + ret = ublk_validate_io_buf(ub, cmd, &auto_buf); if (!ret) + ret = __ublk_fetch(cmd, ub, io, q_id); + if (!ret) { + ublk_apply_io_buf(ub, io, cmd, buf_addr, &auto_buf, NULL); ublk_mark_io_ready(ub, q_id, io); + } mutex_unlock(&ub->mutex); return ret; } @@ -3447,13 +3459,18 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, case UBLK_IO_REGISTER_IO_BUF: return ublk_daemon_register_io_buf(cmd, ub, q_id, tag, io, addr, issue_flags); - case UBLK_IO_COMMIT_AND_FETCH_REQ: + case UBLK_IO_COMMIT_AND_FETCH_REQ: { + struct ublk_auto_buf_reg auto_buf; + ret = ublk_check_commit_and_fetch(ub, io, addr); + if (ret) + goto out; + ret = ublk_validate_io_buf(ub, cmd, &auto_buf); if (ret) goto out; io->res = result; req = ublk_fill_io_cmd(io, cmd); - ret = ublk_config_io_buf(ub, io, cmd, addr, &buf_idx); + ublk_apply_io_buf(ub, io, cmd, addr, &auto_buf, &buf_idx); if (buf_idx != UBLK_INVALID_BUF_IDX) io_buffer_unregister_bvec(cmd, buf_idx, issue_flags); compl = ublk_need_complete_req(ub, io); @@ -3462,10 +3479,8 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, req->__sector = addr; if (compl) __ublk_complete_rq(req, io, ublk_dev_need_map_io(ub), NULL); - - if (ret) - goto out; break; + } case UBLK_IO_NEED_GET_DATA: /* * ublk_get_data() may fail and fallback to requeue, so keep @@ -3473,8 +3488,7 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, * request */ req = ublk_fill_io_cmd(io, cmd); - ret = ublk_config_io_buf(ub, io, cmd, addr, NULL); - WARN_ON_ONCE(ret); + io->buf.addr = addr; if (likely(ublk_get_data(ubq, io, req))) { __ublk_prep_compl_io_cmd(io, req); return UBLK_IO_RES_OK; @@ -3786,11 +3800,11 @@ static int ublk_batch_commit_io(struct ublk_queue *ubq, ret = ublk_batch_commit_io_check(ubq, io, &buf); if (!ret) { io->res = elem->result; - io->buf = buf; req = ublk_fill_io_cmd(io, data->cmd); if (auto_reg) ublk_clear_auto_buf_reg(io, data->cmd, &buf_idx); + io->buf = buf; compl = ublk_need_complete_req(data->ub, io); } ublk_io_unlock(io); @@ -5508,39 +5522,36 @@ static void ublk_unpin_range_pages(unsigned long base_pfn, /* * Inner loop: erase up to UBLK_REMOVE_BATCH matching ranges under - * mas_lock, collecting them into an xarray. Then drop the lock and - * unpin pages + free ranges outside spinlock context. + * mas_lock, collecting the page ranges in a fixed-size array. Then + * drop the lock and unpin pages + free ranges outside spinlock context. * * Returns true if the tree walk completed, false if more ranges remain. - * Xarray key is the base PFN, value encodes nr_pages via xa_mk_value(). */ #define UBLK_REMOVE_BATCH 64 +struct ublk_unpin_range { + unsigned long base_pfn; + unsigned long nr_pages; +}; + static bool __ublk_shmem_remove_ranges(struct ublk_device *ub, int buf_index, int *ret) { MA_STATE(mas, &ub->buf_tree, 0, ULONG_MAX); struct ublk_buf_range *range; - struct xarray to_unpin; - unsigned long idx; + struct ublk_unpin_range to_unpin[UBLK_REMOVE_BATCH]; unsigned int count = 0; + unsigned int i; bool done = false; - void *entry; - - xa_init(&to_unpin); mas_lock(&mas); mas_for_each(&mas, range, ULONG_MAX) { - unsigned long nr; - if (buf_index >= 0 && range->buf_index != buf_index) continue; *ret = 0; - nr = mas.last - mas.index + 1; - if (xa_err(xa_store(&to_unpin, mas.index, - xa_mk_value(nr), GFP_ATOMIC))) - goto unlock; + to_unpin[count].base_pfn = mas.index; + to_unpin[count].nr_pages = mas.last - mas.index + 1; mas_erase(&mas); kfree(range); if (++count >= UBLK_REMOVE_BATCH) @@ -5550,9 +5561,9 @@ static bool __ublk_shmem_remove_ranges(struct ublk_device *ub, unlock: mas_unlock(&mas); - xa_for_each(&to_unpin, idx, entry) - ublk_unpin_range_pages(idx, xa_to_value(entry)); - xa_destroy(&to_unpin); + for (i = 0; i < count; i++) + ublk_unpin_range_pages(to_unpin[i].base_pfn, + to_unpin[i].nr_pages); return done; } diff --git a/drivers/bluetooth/btintel.c b/drivers/bluetooth/btintel.c index bf567b7c5f00b5..cc2234470960a7 100644 --- a/drivers/bluetooth/btintel.c +++ b/drivers/bluetooth/btintel.c @@ -3834,7 +3834,7 @@ int btintel_recv_event(struct hci_dev *hdev, struct sk_buff *skb) /* Handle all diagnostics events separately. May still call * hci_recv_frame. */ - if (len >= sizeof(diagnostics_hdr) && + if (len + 1 >= sizeof(diagnostics_hdr) && memcmp(&skb->data[2], diagnostics_hdr, sizeof(diagnostics_hdr)) == 0) { return btintel_diagnostics(hdev, skb); diff --git a/drivers/bluetooth/btmtk.c b/drivers/bluetooth/btmtk.c index 66b34676104362..c0ed51567ed4dd 100644 --- a/drivers/bluetooth/btmtk.c +++ b/drivers/bluetooth/btmtk.c @@ -860,6 +860,7 @@ static u32 btmtk_usb_reset_done(struct hci_dev *hdev) int btmtk_usb_subsys_reset(struct hci_dev *hdev, u32 dev_id) { + int reset_err = 0; u32 val; int err; @@ -958,8 +959,10 @@ int btmtk_usb_subsys_reset(struct hci_dev *hdev, u32 dev_id) err = readx_poll_timeout(btmtk_usb_reset_done, hdev, val, val & MTK_BT_RST_DONE, 20000, 1000000); - if (err < 0) + if (err < 0) { bt_dev_err(hdev, "Reset timeout"); + reset_err = err; + } if (dev_id == 0x7922) { err = btmtk_usb_uhw_reg_write(hdev, MTK_UDMA_INT_STA_BT, 0x000000FF); @@ -968,10 +971,12 @@ int btmtk_usb_subsys_reset(struct hci_dev *hdev, u32 dev_id) } err = btmtk_usb_id_get(hdev, 0x70010200, &val); - if (err || (!val && dev_id != 0x6639)) + if (err || (!val && dev_id != 0x6639)) { bt_dev_err(hdev, "Can't get device id, subsys reset fail."); + return err ? err : -ENODEV; + } - return err; + return reset_err; } EXPORT_SYMBOL_GPL(btmtk_usb_subsys_reset); diff --git a/drivers/bluetooth/btmtksdio.c b/drivers/bluetooth/btmtksdio.c index c6f80c419e901e..94aa60d9cc207e 100644 --- a/drivers/bluetooth/btmtksdio.c +++ b/drivers/bluetooth/btmtksdio.c @@ -272,12 +272,24 @@ static int btmtksdio_tx_packet(struct btmtksdio_dev *bdev, struct sk_buff *skb) { struct mtkbtsdio_hdr *sdio_hdr; + unsigned int len, pad_len; int err; - /* Make sure that there are enough rooms for SDIO header */ - if (unlikely(skb_headroom(skb) < sizeof(*sdio_hdr))) { - err = pskb_expand_head(skb, sizeof(*sdio_hdr), 0, - GFP_ATOMIC); + /* Make sure that the data buffer is not shared with anyone else and + * that there is enough room for the SDIO header + */ + err = skb_cow_head(skb, sizeof(*sdio_hdr)); + if (err < 0) + return err; + + /* The transfer is rounded up to the SDIO block size, so the buffer + * has to provide tailroom for the padding as well + */ + len = skb->len + sizeof(*sdio_hdr); + pad_len = round_up(len, MTK_SDIO_BLOCK_SIZE) - len; + + if (unlikely(skb_tailroom(skb) < pad_len)) { + err = pskb_expand_head(skb, 0, pad_len, GFP_ATOMIC); if (err < 0) return err; } @@ -290,19 +302,22 @@ static int btmtksdio_tx_packet(struct btmtksdio_dev *bdev, sdio_hdr->reserved = cpu_to_le16(0); sdio_hdr->bt_type = hci_skb_pkt_type(skb); + /* Zero the padding so that no uninitialised memory is sent out */ + skb_put_zero(skb, pad_len); + clear_bit(BTMTKSDIO_HW_TX_READY, &bdev->tx_state); - err = sdio_writesb(bdev->func, MTK_REG_CTDR, skb->data, - round_up(skb->len, MTK_SDIO_BLOCK_SIZE)); + err = sdio_writesb(bdev->func, MTK_REG_CTDR, skb->data, skb->len); if (err < 0) - goto err_skb_pull; + goto err_skb_restore; - bdev->hdev->stat.byte_tx += skb->len; + bdev->hdev->stat.byte_tx += len; kfree_skb(skb); return 0; -err_skb_pull: +err_skb_restore: + skb_trim(skb, len); skb_pull(skb, sizeof(*sdio_hdr)); return err; @@ -1480,6 +1495,9 @@ static void btmtksdio_remove(struct sdio_func *func) if (test_bit(BTMTKSDIO_FUNC_ENABLED, &bdev->tx_state)) btmtksdio_close(hdev); + if (bdev->data->pm_runtime_supported) + pm_runtime_dont_use_autosuspend(bdev->dev); + /* Be consistent the state in btmtksdio_probe */ pm_runtime_get_noresume(bdev->dev); diff --git a/drivers/bluetooth/btnxpuart.c b/drivers/bluetooth/btnxpuart.c index 6a1cffe08d5f7e..e6c15bc6a30b9d 100644 --- a/drivers/bluetooth/btnxpuart.c +++ b/drivers/bluetooth/btnxpuart.c @@ -1368,12 +1368,21 @@ static int nxp_process_fw_dump(struct hci_dev *hdev, struct sk_buff *skb) { struct hci_acl_hdr *acl_hdr = (struct hci_acl_hdr *)skb_pull_data(skb, sizeof(*acl_hdr)); - struct nxp_fw_dump_hdr *fw_dump_hdr = (struct nxp_fw_dump_hdr *)skb->data; + struct nxp_fw_dump_hdr *fw_dump_hdr; struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev); - __u16 seq_num = __le16_to_cpu(fw_dump_hdr->seq_num); - __u16 buf_len = __le16_to_cpu(fw_dump_hdr->buf_len); + __u16 seq_num; + __u16 buf_len; int err; + fw_dump_hdr = skb_pull_data(skb, sizeof(*fw_dump_hdr)); + if (!fw_dump_hdr) { + bt_dev_warn(hdev, "FW dump: invalid or corrupt fw dump chunk"); + goto free_skb; + } + + seq_num = __le16_to_cpu(fw_dump_hdr->seq_num); + buf_len = __le16_to_cpu(fw_dump_hdr->buf_len); + if (seq_num == 0x0001) { if (test_and_set_bit(BTNXPUART_FW_DUMP_IN_PROGRESS, &nxpdev->tx_state)) { bt_dev_err(hdev, "FW dump already in progress"); diff --git a/drivers/bluetooth/btqca.c b/drivers/bluetooth/btqca.c index 10c496eaea2c18..4b0d83858229b1 100644 --- a/drivers/bluetooth/btqca.c +++ b/drivers/bluetooth/btqca.c @@ -1029,8 +1029,7 @@ int qca_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr) baswap(&bdaddr_swapped, bdaddr); skb = __hci_cmd_sync_ev(hdev, EDL_WRITE_BD_ADDR_OPCODE, 6, - &bdaddr_swapped, HCI_EV_VENDOR, - HCI_INIT_TIMEOUT); + &bdaddr_swapped, 0, HCI_INIT_TIMEOUT); if (IS_ERR(skb)) { err = PTR_ERR(skb); bt_dev_err(hdev, "QCA Change address cmd failed (%d)", err); diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index 91b22ddc9f5df0..a3d6d3194b15b6 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -1021,6 +1021,7 @@ struct btusb_data { bool usb_alt6_packet_flow; int isoc_altsetting; int suspend_count; + const struct usb_device_id *match_id; int (*recv_event)(struct hci_dev *hdev, struct sk_buff *skb); int (*recv_acl)(struct hci_dev *hdev, struct sk_buff *skb); @@ -3083,14 +3084,15 @@ static int btusb_set_bdaddr_ath3012(struct hci_dev *hdev, static int btusb_set_bdaddr_wcn6855(struct hci_dev *hdev, const bdaddr_t *bdaddr) { + bdaddr_t bdaddr_swapped; struct sk_buff *skb; - u8 buf[6]; long ret; - memcpy(buf, bdaddr, sizeof(bdaddr_t)); + baswap(&bdaddr_swapped, bdaddr); - skb = __hci_cmd_sync_ev(hdev, 0xfc14, sizeof(buf), buf, - HCI_EV_CMD_COMPLETE, HCI_INIT_TIMEOUT); + skb = __hci_cmd_sync_ev(hdev, 0xfc14, sizeof(bdaddr_swapped), + &bdaddr_swapped, HCI_EV_CMD_COMPLETE, + HCI_INIT_TIMEOUT); if (IS_ERR(skb)) { ret = PTR_ERR(skb); bt_dev_err(hdev, "Change address command failed (%ld)", ret); @@ -3706,8 +3708,10 @@ static int btusb_setup_qca(struct hci_dev *hdev) if (err) return err; - btdata->qca_dump.fw_version = le32_to_cpu(ver.patch_version); - btdata->qca_dump.controller_id = le32_to_cpu(ver.rom_version); + if (btdata->match_id->driver_info & BTUSB_QCA_WCN6855) { + btdata->qca_dump.fw_version = le32_to_cpu(ver.patch_version); + btdata->qca_dump.controller_id = le32_to_cpu(ver.rom_version); + } if (!(status & QCA_SYSCFG_UPDATED)) { err = btusb_setup_qca_load_nvm(hdev, &ver, info); @@ -4118,6 +4122,7 @@ static int btusb_probe(struct usb_interface *intf, if (!data) return -ENOMEM; + data->match_id = id; err = usb_find_common_endpoints(intf->cur_altsetting, &data->bulk_rx_ep, &data->bulk_tx_ep, &data->intr_ep, NULL); if (err) diff --git a/drivers/bluetooth/virtio_bt.c b/drivers/bluetooth/virtio_bt.c index 140ab55c9fc5a9..c20d54088c8c4e 100644 --- a/drivers/bluetooth/virtio_bt.c +++ b/drivers/bluetooth/virtio_bt.c @@ -120,9 +120,13 @@ static int virtbt_setup_zephyr(struct hci_dev *hdev) if (IS_ERR(skb)) return PTR_ERR(skb); - bt_dev_info(hdev, "%s", (char *)(skb->data + 1)); + /* Bounded print: the backend controls skb->len. */ + if (skb->len > 1) { + int len = skb->len - 1; - hci_set_fw_info(hdev, "%s", skb->data + 1); + bt_dev_info(hdev, "%.*s", len, (char *)(skb->data + 1)); + hci_set_fw_info(hdev, "%.*s", len, skb->data + 1); + } kfree_skb(skb); return 0; diff --git a/drivers/bus/mhi/ep/main.c b/drivers/bus/mhi/ep/main.c index b1213786f72c66..21bc2c50170ff7 100644 --- a/drivers/bus/mhi/ep/main.c +++ b/drivers/bus/mhi/ep/main.c @@ -1340,14 +1340,19 @@ static int mhi_ep_create_device(struct mhi_ep_cntrl *mhi_cntrl, u32 ch_id) ret = dev_set_name(&mhi_dev->dev, "%s_%s", dev_name(&mhi_cntrl->mhi_dev->dev), mhi_dev->name); - if (ret) { - put_device(&mhi_dev->dev); - return ret; - } + if (ret) + goto err_put_channels; ret = device_add(&mhi_dev->dev); if (ret) - put_device(&mhi_dev->dev); + goto err_put_channels; + + return 0; + +err_put_channels: + put_device(&mhi_dev->dev); /* DL channel reference */ + put_device(&mhi_dev->dev); /* UL channel reference */ + put_device(&mhi_dev->dev); /* device_initialize() reference */ return ret; } diff --git a/drivers/bus/mhi/host/init.c b/drivers/bus/mhi/host/init.c index 12dcb1a2753c70..fd3050889412de 100644 --- a/drivers/bus/mhi/host/init.c +++ b/drivers/bus/mhi/host/init.c @@ -1029,7 +1029,7 @@ int mhi_register_controller(struct mhi_controller *mhi_cntrl, if (mhi_cntrl->edl_trigger) { ret = sysfs_create_file(&mhi_dev->dev.kobj, &dev_attr_trigger_edl.attr); if (ret) - goto err_release_dev; + goto err_del_dev; } mhi_cntrl->mhi_dev = mhi_dev; @@ -1038,6 +1038,8 @@ int mhi_register_controller(struct mhi_controller *mhi_cntrl, return 0; +err_del_dev: + device_del(&mhi_dev->dev); err_release_dev: put_device(&mhi_dev->dev); error_setup_irq: diff --git a/drivers/bus/mhi/host/main.c b/drivers/bus/mhi/host/main.c index 53c0ffe300702b..4d458396233abc 100644 --- a/drivers/bus/mhi/host/main.c +++ b/drivers/bus/mhi/host/main.c @@ -170,6 +170,9 @@ EXPORT_SYMBOL_GPL(mhi_get_mhi_state); void mhi_soc_reset(struct mhi_controller *mhi_cntrl) { + int __maybe_unused ret; + u32 tmp; + if (mhi_cntrl->reset) { mhi_cntrl->reset(mhi_cntrl); return; @@ -178,6 +181,9 @@ void mhi_soc_reset(struct mhi_controller *mhi_cntrl) /* Generic MHI SoC reset */ mhi_write_reg(mhi_cntrl, mhi_cntrl->regs, MHI_SOC_RESET_REQ_OFFSET, MHI_SOC_RESET_REQ); + /* Flush the posted write to the device (ignore return value) */ + ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->regs, MHI_SOC_RESET_REQ_OFFSET, + &tmp); } EXPORT_SYMBOL_GPL(mhi_soc_reset); diff --git a/drivers/bus/qcom-ebi2.c b/drivers/bus/qcom-ebi2.c index be8166565e7cce..8d2eb955dc9212 100644 --- a/drivers/bus/qcom-ebi2.c +++ b/drivers/bus/qcom-ebi2.c @@ -302,41 +302,23 @@ static int qcom_ebi2_probe(struct platform_device *pdev) u32 val; int ret; - ebi2xclk = devm_clk_get(dev, "ebi2x"); + ebi2xclk = devm_clk_get_enabled(dev, "ebi2x"); if (IS_ERR(ebi2xclk)) return PTR_ERR(ebi2xclk); - ret = clk_prepare_enable(ebi2xclk); - if (ret) { - dev_err(dev, "could not enable EBI2X clk (%d)\n", ret); - return ret; - } - - ebi2clk = devm_clk_get(dev, "ebi2"); - if (IS_ERR(ebi2clk)) { - ret = PTR_ERR(ebi2clk); - goto err_disable_2x_clk; - } - - ret = clk_prepare_enable(ebi2clk); - if (ret) { - dev_err(dev, "could not enable EBI2 clk\n"); - goto err_disable_2x_clk; - } + ebi2clk = devm_clk_get_enabled(dev, "ebi2"); + if (IS_ERR(ebi2clk)) + return PTR_ERR(ebi2clk); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); ebi2_base = devm_ioremap_resource(dev, res); - if (IS_ERR(ebi2_base)) { - ret = PTR_ERR(ebi2_base); - goto err_disable_clk; - } + if (IS_ERR(ebi2_base)) + return PTR_ERR(ebi2_base); res = platform_get_resource(pdev, IORESOURCE_MEM, 1); ebi2_xmem = devm_ioremap_resource(dev, res); - if (IS_ERR(ebi2_xmem)) { - ret = PTR_ERR(ebi2_xmem); - goto err_disable_clk; - } + if (IS_ERR(ebi2_xmem)) + return PTR_ERR(ebi2_xmem); /* Allegedly this turns the power save mode off */ writel(0UL, ebi2_xmem + EBI2_XMEM_CFG); @@ -373,15 +355,9 @@ static int qcom_ebi2_probe(struct platform_device *pdev) } if (have_children) - return of_platform_default_populate(np, NULL, dev); - return 0; + return devm_of_platform_populate(dev); -err_disable_clk: - clk_disable_unprepare(ebi2clk); -err_disable_2x_clk: - clk_disable_unprepare(ebi2xclk); - - return ret; + return 0; } static const struct of_device_id qcom_ebi2_of_match[] = { diff --git a/drivers/bus/ti-sysc.c b/drivers/bus/ti-sysc.c index a5b9507de37c38..e118b900c9ac83 100644 --- a/drivers/bus/ti-sysc.c +++ b/drivers/bus/ti-sysc.c @@ -682,6 +682,7 @@ static struct device_node *stdout_path; static void sysc_init_stdout_path(struct sysc *ddata) { + struct device_node *chosen; struct device_node *np = NULL; const char *uart; @@ -691,15 +692,18 @@ static void sysc_init_stdout_path(struct sysc *ddata) if (stdout_path) return; - np = of_find_node_by_path("/chosen"); - if (!np) + chosen = of_find_node_by_path("/chosen"); + if (!chosen) goto err; - uart = of_get_property(np, "stdout-path", NULL); - if (!uart) + uart = of_get_property(chosen, "stdout-path", NULL); + if (!uart) { + of_node_put(chosen); goto err; + } np = of_find_node_by_path(uart); + of_node_put(chosen); if (!np) goto err; diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 6931657ad2caa8..e77af6578ab503 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -596,11 +596,13 @@ int hwrng_register(struct hwrng *rng) */ err = set_current_rng(rng); if (err) - goto out_unlock; + goto out_list_del; } } mutex_unlock(&rng_mutex); return 0; +out_list_del: + list_del_init(&rng->list); out_unlock: mutex_unlock(&rng_mutex); out: diff --git a/drivers/char/hw_random/imx-rngc.c b/drivers/char/hw_random/imx-rngc.c index 28c56c2d1bf6d9..a2db26d7f790e6 100644 --- a/drivers/char/hw_random/imx-rngc.c +++ b/drivers/char/hw_random/imx-rngc.c @@ -313,8 +313,10 @@ static int __init imx_rngc_probe(struct platform_device *pdev) devm_pm_runtime_enable(&pdev->dev); ret = devm_hwrng_register(&pdev->dev, &rngc->rng); - if (ret) + if (ret) { + clk_disable_unprepare(rngc->clk); return dev_err_probe(&pdev->dev, ret, "hwrng registration failed\n"); + } dev_info(&pdev->dev, "Freescale RNG%c registered (HW revision %d.%02d)\n", diff --git a/drivers/char/hw_random/ks-sa-rng.c b/drivers/char/hw_random/ks-sa-rng.c index 9e408144a10c1e..4494f1e4ab4dbf 100644 --- a/drivers/char/hw_random/ks-sa-rng.c +++ b/drivers/char/hw_random/ks-sa-rng.c @@ -242,7 +242,14 @@ static int ks_sa_rng_probe(struct platform_device *pdev) return dev_err_probe(dev, ret, "Failed to enable SA power-domain\n"); } - return devm_hwrng_register(&pdev->dev, &ks_sa_rng->rng); + ret = devm_hwrng_register(dev, &ks_sa_rng->rng); + if (ret) { + pm_runtime_put_sync(dev); + pm_runtime_disable(dev); + return ret; + } + + return 0; } static void ks_sa_rng_remove(struct platform_device *pdev) diff --git a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c index 5e8b50f15db754..a8c0b3dfb133cd 100644 --- a/drivers/char/hw_random/omap-rng.c +++ b/drivers/char/hw_random/omap-rng.c @@ -455,32 +455,40 @@ static int omap_rng_probe(struct platform_device *pdev) ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) { dev_err(&pdev->dev, "Failed to runtime_get device: %d\n", ret); - goto err_ioremap; + goto err_pm_disable; } priv->clk = devm_clk_get(&pdev->dev, NULL); - if (PTR_ERR(priv->clk) == -EPROBE_DEFER) - return -EPROBE_DEFER; + if (PTR_ERR(priv->clk) == -EPROBE_DEFER) { + ret = -EPROBE_DEFER; + goto err_pm_put; + } if (!IS_ERR(priv->clk)) { ret = clk_prepare_enable(priv->clk); if (ret) { dev_err(&pdev->dev, "Unable to enable the clk: %d\n", ret); - goto err_register; + goto err_pm_put; } + } else { + priv->clk = NULL; } priv->clk_reg = devm_clk_get(&pdev->dev, "reg"); - if (PTR_ERR(priv->clk_reg) == -EPROBE_DEFER) - return -EPROBE_DEFER; + if (PTR_ERR(priv->clk_reg) == -EPROBE_DEFER) { + ret = -EPROBE_DEFER; + goto err_clk; + } if (!IS_ERR(priv->clk_reg)) { ret = clk_prepare_enable(priv->clk_reg); if (ret) { dev_err(&pdev->dev, "Unable to enable the register clk: %d\n", ret); - goto err_register; + goto err_clk; } + } else { + priv->clk_reg = NULL; } ret = (dev->of_node) ? of_get_omap_rng_device_details(priv, pdev) : @@ -498,12 +506,14 @@ static int omap_rng_probe(struct platform_device *pdev) return 0; err_register: + clk_disable_unprepare(priv->clk_reg); +err_clk: + clk_disable_unprepare(priv->clk); +err_pm_put: priv->base = NULL; pm_runtime_put_sync(&pdev->dev); +err_pm_disable: pm_runtime_disable(&pdev->dev); - - clk_disable_unprepare(priv->clk_reg); - clk_disable_unprepare(priv->clk); err_ioremap: dev_err(dev, "initialization failed.\n"); return ret; diff --git a/drivers/char/hw_random/xilinx-trng.c b/drivers/char/hw_random/xilinx-trng.c index 0fbc22c38fbc7f..290bf5bc82db8c 100644 --- a/drivers/char/hw_random/xilinx-trng.c +++ b/drivers/char/hw_random/xilinx-trng.c @@ -86,8 +86,8 @@ static void xtrng_softreset(struct xilinx_rng *rng) xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSRST_MASK, 0); } -/* Return no. of bytes read */ -static size_t xtrng_readblock32(void __iomem *rng_base, __be32 *buf, int blocks32, bool wait) +/* Return no. of bytes read or a negative error before any data is read. */ +static int xtrng_readblock32(void __iomem *rng_base, __be32 *buf, int blocks32, bool wait) { int read = 0, ret; int timeout = 1; @@ -102,8 +102,11 @@ static size_t xtrng_readblock32(void __iomem *rng_base, __be32 *buf, int blocks3 ret = readl_poll_timeout(rng_base + TRNG_STATUS_OFFSET, val, (val & TRNG_STATUS_QCNT_MASK) == TRNG_STATUS_QCNT_16_BYTES, !!wait, timeout); - if (ret) + if (ret) { + if (!read) + return ret; break; + } for (idx = 0; idx < TRNG_READ_4_WORD; idx++) { *(buf + read) = cpu_to_be32(ioread32(rng_base + TRNG_CORE_OUTPUT_OFFSET)); @@ -118,27 +121,40 @@ static int xtrng_collect_random_data(struct xilinx_rng *rng, u8 *rand_gen_buf, { u8 randbuf[TRNG_SEC_STRENGTH_BYTES]; int byteleft, blocks, count = 0; + int full_blocks_bytes; int ret; byteleft = no_of_random_bytes & (TRNG_SEC_STRENGTH_BYTES - 1); blocks = no_of_random_bytes >> TRNG_SEC_STRENGTH_SHIFT; + full_blocks_bytes = blocks * TRNG_SEC_STRENGTH_BYTES; xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSTART_MASK, TRNG_CTRL_PRNGSTART_MASK); if (blocks) { ret = xtrng_readblock32(rng->rng_base, (__be32 *)rand_gen_buf, blocks, wait); - if (!ret) - return 0; + if (ret <= 0) { + count = ret; + goto out_stop; + } count += ret; + if (ret < full_blocks_bytes) + goto out_stop; } if (byteleft) { ret = xtrng_readblock32(rng->rng_base, (__be32 *)randbuf, 1, wait); + if (ret < 0) { + if (!count) + count = ret; + goto out_stop; + } if (!ret) - return count; - memcpy(rand_gen_buf + (blocks * TRNG_SEC_STRENGTH_BYTES), randbuf, byteleft); - count += byteleft; + goto out_stop; + ret = min(ret, no_of_random_bytes - count); + memcpy(rand_gen_buf + count, randbuf, ret); + count += ret; } +out_stop: xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGMODE_MASK | TRNG_CTRL_PRNGSTART_MASK, 0U); diff --git a/drivers/char/ppdev.c b/drivers/char/ppdev.c index 6da817b9849f32..8803268b4cdc88 100644 --- a/drivers/char/ppdev.c +++ b/drivers/char/ppdev.c @@ -340,15 +340,17 @@ static enum ieee1284_phase init_phase(int mode) return IEEE1284_PH_FWD_IDLE; } -static int pp_set_timeout(struct pardevice *pdev, long tv_sec, int tv_usec) +static int pp_set_timeout(struct pardevice *pdev, s64 tv_sec, s64 tv_usec) { + struct timespec64 ts; long to_jiffies; - if ((tv_sec < 0) || (tv_usec < 0)) + if (tv_sec < 0 || tv_usec < 0 || tv_usec >= USEC_PER_SEC) return -EINVAL; - to_jiffies = usecs_to_jiffies(tv_usec); - to_jiffies += tv_sec * HZ; + ts.tv_sec = tv_sec; + ts.tv_nsec = tv_usec * NSEC_PER_USEC; + to_jiffies = timespec64_to_jiffies(&ts); if (to_jiffies <= 0) return -EINVAL; diff --git a/drivers/char/tlclk.c b/drivers/char/tlclk.c index dd45fe5eb6f270..255f69123af5f2 100644 --- a/drivers/char/tlclk.c +++ b/drivers/char/tlclk.c @@ -328,7 +328,7 @@ static DEVICE_ATTR(alarms, S_IRUGO, show_alarms, NULL); static ssize_t store_received_ref_clk3a(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -350,7 +350,7 @@ static DEVICE_ATTR(received_ref_clk3a, (S_IWUSR|S_IWGRP), NULL, static ssize_t store_received_ref_clk3b(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -372,7 +372,7 @@ static DEVICE_ATTR(received_ref_clk3b, (S_IWUSR|S_IWGRP), NULL, static ssize_t store_enable_clk3b_output(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -394,7 +394,7 @@ static ssize_t store_enable_clk3a_output(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { unsigned long flags; - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; sscanf(buf, "%lX", &tmp); @@ -415,7 +415,7 @@ static ssize_t store_enable_clkb1_output(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { unsigned long flags; - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; sscanf(buf, "%lX", &tmp); @@ -437,7 +437,7 @@ static ssize_t store_enable_clka1_output(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { unsigned long flags; - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; sscanf(buf, "%lX", &tmp); @@ -458,7 +458,7 @@ static ssize_t store_enable_clkb0_output(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { unsigned long flags; - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; sscanf(buf, "%lX", &tmp); @@ -479,7 +479,7 @@ static ssize_t store_enable_clka0_output(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { unsigned long flags; - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; sscanf(buf, "%lX", &tmp); @@ -500,7 +500,7 @@ static ssize_t store_select_amcb2_transmit_clock(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { unsigned long flags; - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; sscanf(buf, "%lX", &tmp); @@ -541,7 +541,7 @@ static DEVICE_ATTR(select_amcb2_transmit_clock, (S_IWUSR|S_IWGRP), NULL, static ssize_t store_select_amcb1_transmit_clock(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -583,7 +583,7 @@ static DEVICE_ATTR(select_amcb1_transmit_clock, (S_IWUSR|S_IWGRP), NULL, static ssize_t store_select_redundant_clock(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -604,7 +604,7 @@ static DEVICE_ATTR(select_redundant_clock, (S_IWUSR|S_IWGRP), NULL, static ssize_t store_select_ref_frequency(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -625,7 +625,7 @@ static DEVICE_ATTR(select_ref_frequency, (S_IWUSR|S_IWGRP), NULL, static ssize_t store_filter_select(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -645,7 +645,7 @@ static DEVICE_ATTR(filter_select, (S_IWUSR|S_IWGRP), NULL, store_filter_select); static ssize_t store_hardware_switching_mode(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -666,7 +666,7 @@ static DEVICE_ATTR(hardware_switching_mode, (S_IWUSR|S_IWGRP), NULL, static ssize_t store_hardware_switching(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -687,7 +687,7 @@ static DEVICE_ATTR(hardware_switching, (S_IWUSR|S_IWGRP), NULL, static ssize_t store_refalign (struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned long flags; sscanf(buf, "%lX", &tmp); @@ -706,7 +706,7 @@ static DEVICE_ATTR(refalign, (S_IWUSR|S_IWGRP), NULL, store_refalign); static ssize_t store_mode_select (struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -726,7 +726,7 @@ static DEVICE_ATTR(mode_select, (S_IWUSR|S_IWGRP), NULL, store_mode_select); static ssize_t store_reset (struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; diff --git a/drivers/char/tpm/st33zp24/st33zp24.c b/drivers/char/tpm/st33zp24/st33zp24.c index e2b7451ea7ccd3..0e2deff94c3672 100644 --- a/drivers/char/tpm/st33zp24/st33zp24.c +++ b/drivers/char/tpm/st33zp24/st33zp24.c @@ -93,7 +93,9 @@ static u8 st33zp24_status(struct tpm_chip *chip) struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); u8 data; - tpm_dev->ops->recv(tpm_dev->phy_id, TPM_STS, &data, 1); + if (tpm_dev->ops->recv(tpm_dev->phy_id, TPM_STS, &data, 1) != 1) + return 0; + return data; } @@ -104,10 +106,10 @@ static bool check_locality(struct tpm_chip *chip) { struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); u8 data; - u8 status; + int status; status = tpm_dev->ops->recv(tpm_dev->phy_id, TPM_ACCESS, &data, 1); - if (status && (data & + if (status == 1 && (data & (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) == (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) return true; diff --git a/drivers/char/xilinx_hwicap/xilinx_hwicap.c b/drivers/char/xilinx_hwicap/xilinx_hwicap.c index 34a345dc5e724d..9bb5fa642fd885 100644 --- a/drivers/char/xilinx_hwicap/xilinx_hwicap.c +++ b/drivers/char/xilinx_hwicap/xilinx_hwicap.c @@ -760,7 +760,7 @@ static int __init hwicap_module_init(void) HWICAP_DEVICES, DRIVER_NAME); if (retval < 0) - return retval; + goto failed_class; retval = platform_driver_register(&hwicap_platform_driver); if (retval) @@ -771,6 +771,9 @@ static int __init hwicap_module_init(void) failed: unregister_chrdev_region(devt, HWICAP_DEVICES); + failed_class: + class_unregister(&icap_class); + return retval; } diff --git a/drivers/clk/clk-devres.c b/drivers/clk/clk-devres.c index 994d5bc5168be9..697d102449c109 100644 --- a/drivers/clk/clk-devres.c +++ b/drivers/clk/clk-devres.c @@ -99,6 +99,11 @@ struct clk *devm_clk_get_optional_enabled(struct device *dev, const char *id) } EXPORT_SYMBOL_GPL(devm_clk_get_optional_enabled); +static void devm_clk_disable_unprepare(void *data) +{ + clk_disable_unprepare(data); +} + struct clk *devm_clk_get_optional_enabled_with_rate(struct device *dev, const char *id, unsigned long rate) @@ -106,8 +111,7 @@ struct clk *devm_clk_get_optional_enabled_with_rate(struct device *dev, struct clk *clk; int ret; - clk = __devm_clk_get(dev, id, clk_get_optional, NULL, - clk_disable_unprepare); + clk = devm_clk_get_optional(dev, id); if (IS_ERR(clk)) return ERR_CAST(clk); @@ -119,6 +123,10 @@ struct clk *devm_clk_get_optional_enabled_with_rate(struct device *dev, if (ret) goto out_put_clk; + ret = devm_add_action_or_reset(dev, devm_clk_disable_unprepare, clk); + if (ret) + goto out_put_clk; + return clk; out_put_clk: diff --git a/drivers/clk/clk-k230.c b/drivers/clk/clk-k230.c index cfc437038e4ed0..f34a3e6d3bca01 100644 --- a/drivers/clk/clk-k230.c +++ b/drivers/clk/clk-k230.c @@ -2227,6 +2227,7 @@ static int k230_clk_set_rate_mul(struct clk_hw *hw, unsigned long rate, guard(spinlock)(rate_self->lock); mul_reg = readl(rate_self->reg + clk->mul_reg_off); + mul_reg &= ~(rate_self->mul_mask << rate_self->mul_shift); mul_reg |= ((mul - 1) & rate_self->mul_mask) << (rate_self->mul_shift); mul_reg |= BIT(rate_self->write_enable_bit); writel(mul_reg, rate_self->reg + clk->mul_reg_off); @@ -2257,6 +2258,7 @@ static int k230_clk_set_rate_div(struct clk_hw *hw, unsigned long rate, guard(spinlock)(rate_self->lock); div_reg = readl(rate_self->reg + clk->div_reg_off); + div_reg &= ~(rate_self->div_mask << rate_self->div_shift); div_reg |= ((div - 1) & rate_self->div_mask) << (rate_self->div_shift); div_reg |= BIT(rate_self->write_enable_bit); writel(div_reg, rate_self->reg + clk->div_reg_off); @@ -2287,11 +2289,13 @@ static int k230_clk_set_rate_mul_div(struct clk_hw *hw, unsigned long rate, guard(spinlock)(rate_self->lock); div_reg = readl(rate_self->reg + clk->div_reg_off); + div_reg &= ~(rate_self->div_mask << rate_self->div_shift); div_reg |= ((div - 1) & rate_self->div_mask) << (rate_self->div_shift); div_reg |= BIT(rate_self->write_enable_bit); writel(div_reg, rate_self->reg + clk->div_reg_off); mul_reg = readl(rate_self->reg + clk->mul_reg_off); + mul_reg &= ~(rate_self->mul_mask << rate_self->mul_shift); mul_reg |= ((mul - 1) & rate_self->mul_mask) << (rate_self->mul_shift); mul_reg |= BIT(rate_self->write_enable_bit); writel(mul_reg, rate_self->reg + clk->mul_reg_off); diff --git a/drivers/clk/clk-moxart.c b/drivers/clk/clk-moxart.c index 3786a0153ad17f..40663ef3ef0ae8 100644 --- a/drivers/clk/clk-moxart.c +++ b/drivers/clk/clk-moxart.c @@ -17,7 +17,6 @@ static void __init moxart_of_pll_clk_init(struct device_node *node) { void __iomem *base; struct clk_hw *hw; - struct clk *ref_clk; unsigned int mul; const char *name = node->name; const char *parent_name; @@ -34,12 +33,6 @@ static void __init moxart_of_pll_clk_init(struct device_node *node) mul = readl(base + 0x30) >> 3 & 0x3f; iounmap(base); - ref_clk = of_clk_get(node, 0); - if (IS_ERR(ref_clk)) { - pr_err("%pOF: of_clk_get failed\n", node); - return; - } - hw = clk_hw_register_fixed_factor(NULL, name, parent_name, 0, mul, 1); if (IS_ERR(hw)) { pr_err("%pOF: failed to register clock\n", node); @@ -56,7 +49,6 @@ static void __init moxart_of_apb_clk_init(struct device_node *node) { void __iomem *base; struct clk_hw *hw; - struct clk *pll_clk; unsigned int div, val; unsigned int div_idx[] = { 2, 3, 4, 6, 8}; const char *name = node->name; @@ -78,12 +70,6 @@ static void __init moxart_of_apb_clk_init(struct device_node *node) val = 0; div = div_idx[val] * 2; - pll_clk = of_clk_get(node, 0); - if (IS_ERR(pll_clk)) { - pr_err("%pOF: of_clk_get failed\n", node); - return; - } - hw = clk_hw_register_fixed_factor(NULL, name, parent_name, 0, 1, div); if (IS_ERR(hw)) { pr_err("%pOF: failed to register clock\n", node); diff --git a/drivers/clk/clk-palmas.c b/drivers/clk/clk-palmas.c index 39049f62dbbb30..86a51edac8272a 100644 --- a/drivers/clk/clk-palmas.c +++ b/drivers/clk/clk-palmas.c @@ -194,6 +194,13 @@ static void palmas_clks_get_clk_data(struct platform_device *pdev, cinfo->ext_control_pin = prop; } +static void palmas_clks_unprepare_ext_control(void *data) +{ + struct palmas_clock_info *cinfo = data; + + clk_unprepare(cinfo->hw.clk); +} + static int palmas_clks_init_configure(struct palmas_clock_info *cinfo) { int ret; @@ -214,13 +221,18 @@ static int palmas_clks_init_configure(struct palmas_clock_info *cinfo) return ret; } + ret = devm_add_action_or_reset(cinfo->dev, + palmas_clks_unprepare_ext_control, + cinfo); + if (ret) + return ret; + ret = palmas_ext_control_req_config(cinfo->palmas, cinfo->clk_desc->sleep_reqstr_id, cinfo->ext_control_pin, true); if (ret < 0) { dev_err(cinfo->dev, "Ext config for %s failed, %d\n", cinfo->clk_desc->clk_name, ret); - clk_unprepare(cinfo->hw.clk); return ret; } } diff --git a/drivers/clk/clk-versaclock7.c b/drivers/clk/clk-versaclock7.c index adcc603e325931..e3a36dcd98b806 100644 --- a/drivers/clk/clk-versaclock7.c +++ b/drivers/clk/clk-versaclock7.c @@ -1197,7 +1197,7 @@ static int vc7_probe(struct i2c_client *client) if (ret) { dev_err_probe(&client->dev, ret, "unable to register output %d\n", i); - return ret; + goto err_clk; } switch (bank_src_map.type) { diff --git a/drivers/clk/eswin/clk-eic7700.c b/drivers/clk/eswin/clk-eic7700.c index be81d74192daed..43a47fe16ab14a 100644 --- a/drivers/clk/eswin/clk-eic7700.c +++ b/drivers/clk/eswin/clk-eic7700.c @@ -791,7 +791,8 @@ static struct eswin_clk_info eic7700_clks[] = { EIC7700_CLK_MUX_CPU_ROOT_3MUX1_GFREE, CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_U84, 27, 0), ESWIN_GATE_TYPE(EIC7700_CLK_GATE_NOC_NSP_CLK, "gate_noc_nsp_clk", - EIC7700_CLK_DIV_NOC_NSP_DYNM, CLK_SET_RATE_PARENT, + EIC7700_CLK_DIV_NOC_NSP_DYNM, + CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, EIC7700_REG_OFFSET_NOC, 31, 0), ESWIN_GATE_TYPE(EIC7700_CLK_GATE_BOOTSPI, "gate_clk_bootspi", EIC7700_CLK_MUX_BOOTSPI_CLK_2MUX1_GFREE, diff --git a/drivers/clk/eswin/clk.c b/drivers/clk/eswin/clk.c index e09a52cc358725..79d1e4c5e637a3 100644 --- a/drivers/clk/eswin/clk.c +++ b/drivers/clk/eswin/clk.c @@ -204,7 +204,7 @@ int eswin_clk_register_pll(struct device *dev, struct eswin_pll_clock *clks, int nums, struct eswin_clock_data *data) { struct eswin_clk_pll *p_clk = NULL; - struct clk_init_data init; + struct clk_init_data init = {}; struct clk_hw *clk_hw; int i, ret; @@ -419,7 +419,7 @@ struct clk_hw *eswin_register_clkdiv(struct device *dev, unsigned int id, unsigned long priv_flag, spinlock_t *lock) { struct eswin_divider_clock *dclk; - struct clk_init_data init; + struct clk_init_data init = {}; struct clk_hw *clk_hw; int ret; diff --git a/drivers/clk/hisilicon/reset.c b/drivers/clk/hisilicon/reset.c index 93cee17db8b160..c3b7daac931320 100644 --- a/drivers/clk/hisilicon/reset.c +++ b/drivers/clk/hisilicon/reset.c @@ -91,7 +91,7 @@ struct hisi_reset_controller *hisi_reset_init(struct platform_device *pdev) { struct hisi_reset_controller *rstc; - rstc = devm_kmalloc(&pdev->dev, sizeof(*rstc), GFP_KERNEL); + rstc = devm_kzalloc(&pdev->dev, sizeof(*rstc), GFP_KERNEL); if (!rstc) return NULL; diff --git a/drivers/clk/imx/clk-scu.c b/drivers/clk/imx/clk-scu.c index 9b33df9967ece4..658b6d94de9b45 100644 --- a/drivers/clk/imx/clk-scu.c +++ b/drivers/clk/imx/clk-scu.c @@ -475,7 +475,6 @@ struct clk_hw *__imx_clk_scu(struct device *dev, const char *name, clk->clk_type = clk_type; init.name = name; - init.ops = &clk_scu_ops; if (rsrc_id == IMX_SC_R_A35 || rsrc_id == IMX_SC_R_A53 || rsrc_id == IMX_SC_R_A72) init.ops = &clk_scu_cpu_ops; else if (rsrc_id == IMX_SC_R_PI_0_PLL) diff --git a/drivers/clk/mediatek/clk-mt6735-apmixedsys.c b/drivers/clk/mediatek/clk-mt6735-apmixedsys.c index 9e30c089a20924..b6eb6a581c31e9 100644 --- a/drivers/clk/mediatek/clk-mt6735-apmixedsys.c +++ b/drivers/clk/mediatek/clk-mt6735-apmixedsys.c @@ -102,9 +102,12 @@ static int clk_mt6735_apmixed_probe(struct platform_device *pdev) ret = devm_of_clk_add_hw_provider(&pdev->dev, of_clk_hw_onecell_get, clk_data); - if (ret) + if (ret) { dev_err(&pdev->dev, "Failed to register clock provider: %d\n", ret); + mtk_clk_unregister_plls(apmixedsys_plls, ARRAY_SIZE(apmixedsys_plls), + clk_data); + } return ret; } diff --git a/drivers/clk/mediatek/clk-mt8135.c b/drivers/clk/mediatek/clk-mt8135.c index 084e48a554c262..1d20e15608f77d 100644 --- a/drivers/clk/mediatek/clk-mt8135.c +++ b/drivers/clk/mediatek/clk-mt8135.c @@ -409,6 +409,9 @@ static const struct mtk_gate_regs infra_cg_regs = { GATE_MTK_FLAGS(_id, _name, _parent, &infra_cg_regs, _shift, \ &mtk_clk_gate_ops_setclr, CLK_IS_CRITICAL) +#define GATE_ICG_INV(_id, _name, _parent, _shift) \ + GATE_MTK(_id, _name, _parent, &infra_cg_regs, _shift, &mtk_clk_gate_ops_setclr_inv) + static const struct mtk_gate infra_clks[] = { GATE_DUMMY(CLK_DUMMY, "infra_dummy"), GATE_ICG(CLK_INFRA_PMIC_WRAP, "pmic_wrap_ck", "axi_sel", 23), @@ -419,7 +422,7 @@ static const struct mtk_gate infra_clks[] = { GATE_ICG(CLK_INFRA_CPUM, "cpum_ck", "cpum_tck_in", 15), GATE_ICG_AO(CLK_INFRA_M4U, "m4u_ck", "mem_sel", 8), GATE_ICG(CLK_INFRA_MFGAXI, "mfgaxi_ck", "axi_sel", 7), - GATE_ICG(CLK_INFRA_DEVAPC, "devapc_ck", "axi_sel", 6), + GATE_ICG_INV(CLK_INFRA_DEVAPC, "devapc_ck", "axi_sel", 6), GATE_ICG(CLK_INFRA_AUDIO, "audio_ck", "aud_intbus_sel", 5), GATE_ICG(CLK_INFRA_MFG_BUS, "mfg_bus_ck", "axi_sel", 2), GATE_ICG(CLK_INFRA_SMI, "smi_ck", "smi_sel", 1), diff --git a/drivers/clk/mediatek/clk-pllfh.c b/drivers/clk/mediatek/clk-pllfh.c index aa95cd9197b3c1..6249fb87b1f53b 100644 --- a/drivers/clk/mediatek/clk-pllfh.c +++ b/drivers/clk/mediatek/clk-pllfh.c @@ -197,12 +197,56 @@ static void mtk_clk_unregister_pllfh(struct clk_hw *hw) kfree(fh); } +static void mtk_clk_cleanup_pllfhs(void __iomem *iomem_base, + const struct mtk_pll_data *plls, int num_plls, + void __iomem *iomem_fhctl_base, + struct mtk_pllfh_data *pllfhs, int num_fhs, + struct clk_hw_onecell_data *clk_data) +{ + void __iomem *base = iomem_base; + void __iomem *fhctl_base = iomem_fhctl_base; + int i; + + for (i = num_plls - 1; i >= 0; i--) { + const struct mtk_pll_data *pll = &plls[i]; + struct mtk_pllfh_data *pllfh; + bool use_fhctl; + + if (IS_ERR_OR_NULL(clk_data->hws[pll->id])) + continue; + + pllfh = get_pllfh_by_id(pllfhs, num_fhs, pll->id); + use_fhctl = fhctl_is_supported_and_enabled(pllfh); + + if (!base) + base = mtk_clk_pll_get_base(clk_data->hws[pll->id], + pll); + + if (use_fhctl) { + if (!fhctl_base) + fhctl_base = pllfh->state.base; + mtk_clk_unregister_pllfh(clk_data->hws[pll->id]); + } else { + mtk_clk_unregister_pll(clk_data->hws[pll->id]); + } + + clk_data->hws[pll->id] = ERR_PTR(-ENOENT); + } + + if (fhctl_base) + iounmap(fhctl_base); + + if (base) + iounmap(base); +} + + int mtk_clk_register_pllfhs(struct device *dev, const struct mtk_pll_data *plls, int num_plls, struct mtk_pllfh_data *pllfhs, int num_fhs, struct clk_hw_onecell_data *clk_data) { - void __iomem *base; + void __iomem *base, *fhctl_base = NULL; int i; struct clk_hw *hw; @@ -238,24 +282,8 @@ int mtk_clk_register_pllfhs(struct device *dev, return 0; err: - while (--i >= 0) { - const struct mtk_pll_data *pll = &plls[i]; - struct mtk_pllfh_data *pllfh; - bool use_fhctl; - - pllfh = get_pllfh_by_id(pllfhs, num_fhs, pll->id); - use_fhctl = fhctl_is_supported_and_enabled(pllfh); - - if (use_fhctl) - mtk_clk_unregister_pllfh(clk_data->hws[pll->id]); - else - mtk_clk_unregister_pll(clk_data->hws[pll->id]); - - clk_data->hws[pll->id] = ERR_PTR(-ENOENT); - } - - iounmap(base); - + mtk_clk_cleanup_pllfhs(base, plls, i, fhctl_base, pllfhs, num_fhs, + clk_data); return PTR_ERR(hw); } EXPORT_SYMBOL_GPL(mtk_clk_register_pllfhs); @@ -264,38 +292,10 @@ void mtk_clk_unregister_pllfhs(const struct mtk_pll_data *plls, int num_plls, struct mtk_pllfh_data *pllfhs, int num_fhs, struct clk_hw_onecell_data *clk_data) { - void __iomem *base = NULL, *fhctl_base = NULL; - int i; - if (!clk_data) return; - for (i = num_plls; i > 0; i--) { - const struct mtk_pll_data *pll = &plls[i - 1]; - struct mtk_pllfh_data *pllfh; - bool use_fhctl; - - if (IS_ERR_OR_NULL(clk_data->hws[pll->id])) - continue; - - pllfh = get_pllfh_by_id(pllfhs, num_fhs, pll->id); - use_fhctl = fhctl_is_supported_and_enabled(pllfh); - - if (use_fhctl) { - fhctl_base = pllfh->state.base; - mtk_clk_unregister_pllfh(clk_data->hws[pll->id]); - } else { - base = mtk_clk_pll_get_base(clk_data->hws[pll->id], - pll); - mtk_clk_unregister_pll(clk_data->hws[pll->id]); - } - - clk_data->hws[pll->id] = ERR_PTR(-ENOENT); - } - - if (fhctl_base) - iounmap(fhctl_base); - - iounmap(base); + mtk_clk_cleanup_pllfhs(NULL, plls, num_plls, NULL, pllfhs, + num_fhs, clk_data); } EXPORT_SYMBOL_GPL(mtk_clk_unregister_pllfhs); diff --git a/drivers/clk/nuvoton/clk-ma35d1-pll.c b/drivers/clk/nuvoton/clk-ma35d1-pll.c index 4620acfe47e85f..c7c0dc91a012c7 100644 --- a/drivers/clk/nuvoton/clk-ma35d1-pll.c +++ b/drivers/clk/nuvoton/clk-ma35d1-pll.c @@ -48,7 +48,7 @@ #define PLL_CTL1_PD BIT(0) #define PLL_CTL1_BP BIT(1) #define PLL_CTL1_OUTDIV GENMASK(6, 4) -#define PLL_CTL1_FRAC GENMASK(31, 24) +#define PLL_CTL1_FRAC GENMASK(31, 8) #define PLL_CTL2_SLOPE GENMASK(23, 0) #define INDIV_MIN 1 @@ -92,7 +92,7 @@ static unsigned long ma35d1_calc_smic_pll_freq(u32 pll0_ctl0, p = FIELD_GET(SPLL0_CTL0_OUTDIV, pll0_ctl0); outdiv = 1 << p; pll_freq = (u64)parent_rate * n; - div_u64(pll_freq, m * outdiv); + pll_freq = div_u64(pll_freq, m * outdiv); return pll_freq; } @@ -110,12 +110,12 @@ static unsigned long ma35d1_calc_pll_freq(u8 mode, u32 *reg_ctl, unsigned long p if (mode == PLL_MODE_INT) { pll_freq = (u64)parent_rate * n; - div_u64(pll_freq, m * p); + pll_freq = div_u64(pll_freq, m * p); } else { x = FIELD_GET(PLL_CTL1_FRAC, reg_ctl[1]); - /* 2 decimal places floating to integer (ex. 1.23 to 123) */ - n = n * 100 + ((x * 100) / FIELD_MAX(PLL_CTL1_FRAC)); - pll_freq = div_u64(parent_rate * n, 100 * m * p); + /* convert 24-bit fraction to 3 decimal digits, rounding to closest */ + n = n * 1000 + DIV_ROUND_CLOSEST_ULL((u64)x * 1000, 1ULL << 24); + pll_freq = div_u64((u64)parent_rate * n, 1000 * m * p); } return pll_freq; } @@ -255,32 +255,32 @@ static int ma35d1_clk_pll_determine_rate(struct clk_hw *hw, if (req->best_parent_rate < PLL_FREF_MIN_FREQ || req->best_parent_rate > PLL_FREF_MAX_FREQ) return -EINVAL; - ret = ma35d1_pll_find_closest(pll, req->rate, req->best_parent_rate, - reg_ctl, &pll_freq); - if (ret < 0) - return ret; - switch (pll->id) { case CAPLL: + case DDRPLL: + /* Read-only PLLs: return current rate */ reg_ctl[0] = readl_relaxed(pll->ctl0_base); - pll_freq = ma35d1_calc_smic_pll_freq(reg_ctl[0], req->best_parent_rate); + if (pll->id == CAPLL) { + pll_freq = ma35d1_calc_smic_pll_freq(reg_ctl[0], req->best_parent_rate); + } else { + reg_ctl[1] = readl_relaxed(pll->ctl1_base); + pll_freq = ma35d1_calc_pll_freq(pll->mode, reg_ctl, req->best_parent_rate); + } req->rate = pll_freq; - return 0; - case DDRPLL: case APLL: case EPLL: case VPLL: - reg_ctl[0] = readl_relaxed(pll->ctl0_base); - reg_ctl[1] = readl_relaxed(pll->ctl1_base); - pll_freq = ma35d1_calc_pll_freq(pll->mode, reg_ctl, req->best_parent_rate); + /* Configurable PLLs: find closest achievable rate */ + ret = ma35d1_pll_find_closest(pll, req->rate, req->best_parent_rate, + reg_ctl, &pll_freq); + if (ret < 0) + return ret; req->rate = pll_freq; - return 0; } req->rate = 0; - return 0; } diff --git a/drivers/clk/qcom/camcc-sc8280xp.c b/drivers/clk/qcom/camcc-sc8280xp.c index e97b8d4f3c844e..660d8655d3919a 100644 --- a/drivers/clk/qcom/camcc-sc8280xp.c +++ b/drivers/clk/qcom/camcc-sc8280xp.c @@ -1753,24 +1753,6 @@ static struct clk_branch camcc_csiphy3_clk = { }, }; -static struct clk_branch camcc_gdsc_clk = { - .halt_reg = 0xc1e4, - .halt_check = BRANCH_HALT, - .clkr = { - .enable_reg = 0xc1e4, - .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ - .name = "camcc_gdsc_clk", - .parent_hws = (const struct clk_hw*[]){ - &camcc_xo_clk_src.clkr.hw, - }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch camcc_icp_ahb_clk = { .halt_reg = 0xc0d8, .halt_check = BRANCH_HALT, @@ -2839,7 +2821,6 @@ static struct clk_regmap *camcc_sc8280xp_clocks[] = { [CAMCC_CSIPHY2_CLK] = &camcc_csiphy2_clk.clkr, [CAMCC_CSIPHY3_CLK] = &camcc_csiphy3_clk.clkr, [CAMCC_FAST_AHB_CLK_SRC] = &camcc_fast_ahb_clk_src.clkr, - [CAMCC_GDSC_CLK] = &camcc_gdsc_clk.clkr, [CAMCC_ICP_AHB_CLK] = &camcc_icp_ahb_clk.clkr, [CAMCC_ICP_CLK] = &camcc_icp_clk.clkr, [CAMCC_ICP_CLK_SRC] = &camcc_icp_clk_src.clkr, diff --git a/drivers/clk/qcom/common.c b/drivers/clk/qcom/common.c index eec369d2173b5c..0e8f380873af0f 100644 --- a/drivers/clk/qcom/common.c +++ b/drivers/clk/qcom/common.c @@ -169,7 +169,7 @@ static int _qcom_cc_register_board_clk(struct device *dev, const char *path, if (!node) { fixed = devm_kzalloc(dev, sizeof(*fixed), GFP_KERNEL); if (!fixed) - return -EINVAL; + return -ENOMEM; fixed->fixed_rate = rate; fixed->hw.init = &init_data; @@ -186,7 +186,7 @@ static int _qcom_cc_register_board_clk(struct device *dev, const char *path, if (add_factor) { factor = devm_kzalloc(dev, sizeof(*factor), GFP_KERNEL); if (!factor) - return -EINVAL; + return -ENOMEM; factor->mult = factor->div = 1; factor->hw.init = &init_data; diff --git a/drivers/clk/qcom/dispcc-qcm2290.c b/drivers/clk/qcom/dispcc-qcm2290.c index 4d6aad280ae178..d32fd3f8d7acda 100644 --- a/drivers/clk/qcom/dispcc-qcm2290.c +++ b/drivers/clk/qcom/dispcc-qcm2290.c @@ -2,6 +2,7 @@ /* * Copyright (c) 2020, The Linux Foundation. All rights reserved. * Copyright (c) 2021, Linaro Ltd. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -48,6 +49,7 @@ static const struct alpha_pll_config disp_cc_pll0_config = { static struct clk_alpha_pll disp_cc_pll0 = { .offset = 0x0, + .config = &disp_cc_pll0_config, .vco_table = spark_vco, .num_vco = ARRAY_SIZE(spark_vco), .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], @@ -449,11 +451,14 @@ static const struct qcom_reset_map disp_cc_qcm2290_resets[] = { static struct gdsc mdss_gdsc = { .gdscr = 0x3000, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, .pd = { .name = "mdss_gdsc", }, .pwrsts = PWRSTS_OFF_ON, - .flags = HW_CTRL, + .flags = HW_CTRL | POLL_CFG_GDSCR | RETAIN_FF_ENABLE, }; static struct gdsc *disp_cc_qcm2290_gdscs[] = { @@ -482,6 +487,14 @@ static struct clk_regmap *disp_cc_qcm2290_clocks[] = { [DISP_CC_SLEEP_CLK_SRC] = &disp_cc_sleep_clk_src.clkr, }; +static struct clk_alpha_pll *disp_cc_qcm2290_plls[] = { + &disp_cc_pll0, +}; + +static const u32 disp_cc_qcm2290_critical_cbcrs[] = { + 0x604c, /* DISP_CC_XO_CLK */ +}; + static const struct regmap_config disp_cc_qcm2290_regmap_config = { .reg_bits = 32, .reg_stride = 4, @@ -490,6 +503,13 @@ static const struct regmap_config disp_cc_qcm2290_regmap_config = { .fast_io = true, }; +static const struct qcom_cc_driver_data disp_cc_qcm2290_driver_data = { + .alpha_plls = disp_cc_qcm2290_plls, + .num_alpha_plls = ARRAY_SIZE(disp_cc_qcm2290_plls), + .clk_cbcrs = disp_cc_qcm2290_critical_cbcrs, + .num_clk_cbcrs = ARRAY_SIZE(disp_cc_qcm2290_critical_cbcrs), +}; + static const struct qcom_cc_desc disp_cc_qcm2290_desc = { .config = &disp_cc_qcm2290_regmap_config, .clks = disp_cc_qcm2290_clocks, @@ -498,6 +518,8 @@ static const struct qcom_cc_desc disp_cc_qcm2290_desc = { .num_gdscs = ARRAY_SIZE(disp_cc_qcm2290_gdscs), .resets = disp_cc_qcm2290_resets, .num_resets = ARRAY_SIZE(disp_cc_qcm2290_resets), + .use_rpm = true, + .driver_data = &disp_cc_qcm2290_driver_data, }; static const struct of_device_id disp_cc_qcm2290_match_table[] = { @@ -508,25 +530,7 @@ MODULE_DEVICE_TABLE(of, disp_cc_qcm2290_match_table); static int disp_cc_qcm2290_probe(struct platform_device *pdev) { - struct regmap *regmap; - int ret; - - regmap = qcom_cc_map(pdev, &disp_cc_qcm2290_desc); - if (IS_ERR(regmap)) - return PTR_ERR(regmap); - - clk_alpha_pll_configure(&disp_cc_pll0, regmap, &disp_cc_pll0_config); - - /* Keep some clocks always-on */ - qcom_branch_set_clk_en(regmap, 0x604c); /* DISP_CC_XO_CLK */ - - ret = qcom_cc_really_probe(&pdev->dev, &disp_cc_qcm2290_desc, regmap); - if (ret) { - dev_err(&pdev->dev, "Failed to register DISP CC clocks\n"); - return ret; - } - - return ret; + return qcom_cc_probe(pdev, &disp_cc_qcm2290_desc); } static struct platform_driver disp_cc_qcm2290_driver = { diff --git a/drivers/clk/qcom/gcc-glymur.c b/drivers/clk/qcom/gcc-glymur.c index f4ede4a3a1c07c..2ee4820b6fdfb1 100644 --- a/drivers/clk/qcom/gcc-glymur.c +++ b/drivers/clk/qcom/gcc-glymur.c @@ -3668,21 +3668,6 @@ static struct clk_branch gcc_disp_hf_axi_clk = { }, }; -static struct clk_branch gcc_eva_ahb_clk = { - .halt_reg = 0x9b004, - .halt_check = BRANCH_HALT_VOTED, - .hwcg_reg = 0x9b004, - .hwcg_bit = 1, - .clkr = { - .enable_reg = 0x9b004, - .enable_mask = BIT(0), - .hw.init = &(const struct clk_init_data) { - .name = "gcc_eva_ahb_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch gcc_eva_axi0_clk = { .halt_reg = 0x9b008, .halt_check = BRANCH_HALT_SKIP, @@ -3713,19 +3698,6 @@ static struct clk_branch gcc_eva_axi0c_clk = { }, }; -static struct clk_branch gcc_eva_xo_clk = { - .halt_reg = 0x9b024, - .halt_check = BRANCH_HALT, - .clkr = { - .enable_reg = 0x9b024, - .enable_mask = BIT(0), - .hw.init = &(const struct clk_init_data) { - .name = "gcc_eva_xo_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch gcc_gp1_clk = { .halt_reg = 0x64000, .halt_check = BRANCH_HALT, @@ -7992,10 +7964,8 @@ static struct clk_regmap *gcc_glymur_clocks[] = { [GCC_CFG_NOC_USB_ANOC_AHB_CLK] = &gcc_cfg_noc_usb_anoc_ahb_clk.clkr, [GCC_CFG_NOC_USB_ANOC_SOUTH_AHB_CLK] = &gcc_cfg_noc_usb_anoc_south_ahb_clk.clkr, [GCC_DISP_HF_AXI_CLK] = &gcc_disp_hf_axi_clk.clkr, - [GCC_EVA_AHB_CLK] = &gcc_eva_ahb_clk.clkr, [GCC_EVA_AXI0_CLK] = &gcc_eva_axi0_clk.clkr, [GCC_EVA_AXI0C_CLK] = &gcc_eva_axi0c_clk.clkr, - [GCC_EVA_XO_CLK] = &gcc_eva_xo_clk.clkr, [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, [GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr, [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, @@ -8544,6 +8514,8 @@ static const u32 gcc_glymur_critical_cbcrs[] = { 0x71004, /* GCC_GPU_CFG_AHB_CLK */ 0x32004, /* GCC_VIDEO_AHB_CLK */ 0x32058, /* GCC_VIDEO_XO_CLK */ + 0x9b004, /* GCC_EVA_AHB_CLK */ + 0x9b024, /* GCC_EVA_XO_CLK */ }; static const struct regmap_config gcc_glymur_regmap_config = { @@ -8576,6 +8548,7 @@ static const struct qcom_cc_desc gcc_glymur_desc = { .num_resets = ARRAY_SIZE(gcc_glymur_resets), .gdscs = gcc_glymur_gdscs, .num_gdscs = ARRAY_SIZE(gcc_glymur_gdscs), + .use_rpm = true, .driver_data = &gcc_glymur_driver_data, }; diff --git a/drivers/clk/qcom/gcc-nord.c b/drivers/clk/qcom/gcc-nord.c index 7c7c2171ac965f..5c9d25f53a6d25 100644 --- a/drivers/clk/qcom/gcc-nord.c +++ b/drivers/clk/qcom/gcc-nord.c @@ -701,7 +701,7 @@ static struct clk_branch gcc_pcie_a_phy_rchng_clk = { static struct clk_branch gcc_pcie_a_pipe_clk = { .halt_reg = 0x49068, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .hwcg_reg = 0x49068, .hwcg_bit = 1, .clkr = { @@ -850,7 +850,7 @@ static struct clk_branch gcc_pcie_b_phy_rchng_clk = { static struct clk_branch gcc_pcie_b_pipe_clk = { .halt_reg = 0x4a068, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x9d008, .enable_mask = BIT(24), @@ -995,7 +995,7 @@ static struct clk_branch gcc_pcie_c_phy_rchng_clk = { static struct clk_branch gcc_pcie_c_pipe_clk = { .halt_reg = 0x4b068, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x9d010, .enable_mask = BIT(1), @@ -1140,7 +1140,7 @@ static struct clk_branch gcc_pcie_d_phy_rchng_clk = { static struct clk_branch gcc_pcie_d_pipe_clk = { .halt_reg = 0x4c068, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x9d010, .enable_mask = BIT(10), @@ -1184,34 +1184,6 @@ static struct clk_branch gcc_pcie_d_slv_q2a_axi_clk = { }, }; -static struct clk_branch gcc_pcie_link_ahb_clk = { - .halt_reg = 0x52464, - .halt_check = BRANCH_HALT, - .clkr = { - .enable_reg = 0x52464, - .enable_mask = BIT(0), - .hw.init = &(const struct clk_init_data) { - .name = "gcc_pcie_link_ahb_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - -static struct clk_branch gcc_pcie_link_xo_clk = { - .halt_reg = 0x52468, - .halt_check = BRANCH_HALT_VOTED, - .hwcg_reg = 0x52468, - .hwcg_bit = 1, - .clkr = { - .enable_reg = 0x52468, - .enable_mask = BIT(0), - .hw.init = &(const struct clk_init_data) { - .name = "gcc_pcie_link_xo_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch gcc_pcie_noc_async_bridge_clk = { .halt_reg = 0x52048, .halt_check = BRANCH_HALT_SKIP, @@ -1757,8 +1729,6 @@ static struct clk_regmap *gcc_nord_clocks[] = { [GCC_PCIE_D_PIPE_CLK_SRC] = &gcc_pcie_d_pipe_clk_src.clkr, [GCC_PCIE_D_SLV_AXI_CLK] = &gcc_pcie_d_slv_axi_clk.clkr, [GCC_PCIE_D_SLV_Q2A_AXI_CLK] = &gcc_pcie_d_slv_q2a_axi_clk.clkr, - [GCC_PCIE_LINK_AHB_CLK] = &gcc_pcie_link_ahb_clk.clkr, - [GCC_PCIE_LINK_XO_CLK] = &gcc_pcie_link_xo_clk.clkr, [GCC_PCIE_NOC_ASYNC_BRIDGE_CLK] = &gcc_pcie_noc_async_bridge_clk.clkr, [GCC_PCIE_NOC_CNOC_SF_QX_CLK] = &gcc_pcie_noc_cnoc_sf_qx_clk.clkr, [GCC_PCIE_NOC_M_CFG_CLK] = &gcc_pcie_noc_m_cfg_clk.clkr, @@ -1849,9 +1819,16 @@ static const struct regmap_config gcc_nord_regmap_config = { .fast_io = true, }; +static const u32 gcc_nord_critical_cbcrs[] = { + 0x52464, /* GCC_PCIE_LINK_AHB_CLK */ + 0x52468, /* GCC_PCIE_LINK_XO_CLK */ +}; + static const struct qcom_cc_driver_data gcc_nord_driver_data = { .dfs_rcgs = gcc_nord_dfs_clocks, .num_dfs_rcgs = ARRAY_SIZE(gcc_nord_dfs_clocks), + .clk_cbcrs = gcc_nord_critical_cbcrs, + .num_clk_cbcrs = ARRAY_SIZE(gcc_nord_critical_cbcrs), }; static const struct qcom_cc_desc gcc_nord_desc = { diff --git a/drivers/clk/qcom/gcc-qcm2290.c b/drivers/clk/qcom/gcc-qcm2290.c index 6684cab63ae116..77cff0e9af93be 100644 --- a/drivers/clk/qcom/gcc-qcm2290.c +++ b/drivers/clk/qcom/gcc-qcm2290.c @@ -116,7 +116,7 @@ static const struct alpha_pll_config gpll10_config = { .vco_mask = GENMASK(21, 20), .main_output_mask = BIT(0), .config_ctl_val = 0x4001055B, - .test_ctl_hi1_val = 0x1, + .test_ctl_hi_val = 0x1, }; static struct clk_alpha_pll gpll10 = { @@ -148,7 +148,7 @@ static const struct alpha_pll_config gpll11_config = { .vco_mask = GENMASK(21, 20), .main_output_mask = BIT(0), .config_ctl_val = 0x4001055B, - .test_ctl_hi1_val = 0x1, + .test_ctl_hi_val = 0x1, }; static struct clk_alpha_pll gpll11 = { @@ -309,7 +309,7 @@ static const struct alpha_pll_config gpll8_config = { .post_div_val = 0x1 << 8, .post_div_mask = GENMASK(11, 8), .config_ctl_val = 0x4001055B, - .test_ctl_hi1_val = 0x1, + .test_ctl_hi_val = 0x1, }; static struct clk_alpha_pll gpll8 = { @@ -1082,7 +1082,7 @@ static struct clk_init_data gcc_qupv3_wrap0_s0_clk_src_init = { .name = "gcc_qupv3_wrap0_s0_clk_src", .parent_data = gcc_parents_1, .num_parents = ARRAY_SIZE(gcc_parents_1), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_no_init_park_ops, }; static struct clk_rcg2 gcc_qupv3_wrap0_s0_clk_src = { @@ -1098,7 +1098,7 @@ static struct clk_init_data gcc_qupv3_wrap0_s1_clk_src_init = { .name = "gcc_qupv3_wrap0_s1_clk_src", .parent_data = gcc_parents_1, .num_parents = ARRAY_SIZE(gcc_parents_1), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_no_init_park_ops, }; static struct clk_rcg2 gcc_qupv3_wrap0_s1_clk_src = { @@ -1114,7 +1114,7 @@ static struct clk_init_data gcc_qupv3_wrap0_s2_clk_src_init = { .name = "gcc_qupv3_wrap0_s2_clk_src", .parent_data = gcc_parents_1, .num_parents = ARRAY_SIZE(gcc_parents_1), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_no_init_park_ops, }; static struct clk_rcg2 gcc_qupv3_wrap0_s2_clk_src = { @@ -1130,7 +1130,7 @@ static struct clk_init_data gcc_qupv3_wrap0_s3_clk_src_init = { .name = "gcc_qupv3_wrap0_s3_clk_src", .parent_data = gcc_parents_1, .num_parents = ARRAY_SIZE(gcc_parents_1), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_no_init_park_ops, }; static struct clk_rcg2 gcc_qupv3_wrap0_s3_clk_src = { @@ -1146,7 +1146,7 @@ static struct clk_init_data gcc_qupv3_wrap0_s4_clk_src_init = { .name = "gcc_qupv3_wrap0_s4_clk_src", .parent_data = gcc_parents_1, .num_parents = ARRAY_SIZE(gcc_parents_1), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_no_init_park_ops, }; static struct clk_rcg2 gcc_qupv3_wrap0_s4_clk_src = { @@ -1162,7 +1162,7 @@ static struct clk_init_data gcc_qupv3_wrap0_s5_clk_src_init = { .name = "gcc_qupv3_wrap0_s5_clk_src", .parent_data = gcc_parents_1, .num_parents = ARRAY_SIZE(gcc_parents_1), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_no_init_park_ops, }; static struct clk_rcg2 gcc_qupv3_wrap0_s5_clk_src = { diff --git a/drivers/clk/qcom/gcc-qcs8300.c b/drivers/clk/qcom/gcc-qcs8300.c index 07218d9c96a7d6..31fd870b10f7ae 100644 --- a/drivers/clk/qcom/gcc-qcs8300.c +++ b/drivers/clk/qcom/gcc-qcs8300.c @@ -3267,7 +3267,7 @@ static struct gdsc gcc_pcie_0_gdsc = { .pd = { .name = "gcc_pcie_0_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = VOTABLE | RETAIN_FF_ENABLE | POLL_CFG_GDSCR, }; @@ -3281,7 +3281,7 @@ static struct gdsc gcc_pcie_1_gdsc = { .pd = { .name = "gcc_pcie_1_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = VOTABLE | RETAIN_FF_ENABLE | POLL_CFG_GDSCR, }; @@ -3305,7 +3305,7 @@ static struct gdsc gcc_usb20_prim_gdsc = { .pd = { .name = "gcc_usb20_prim_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = RETAIN_FF_ENABLE | POLL_CFG_GDSCR, }; @@ -3317,7 +3317,7 @@ static struct gdsc gcc_usb30_prim_gdsc = { .pd = { .name = "gcc_usb30_prim_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = RETAIN_FF_ENABLE | POLL_CFG_GDSCR, }; diff --git a/drivers/clk/qcom/gcc-sm6115.c b/drivers/clk/qcom/gcc-sm6115.c index 4c3804701e2437..4bd2ebc1589464 100644 --- a/drivers/clk/qcom/gcc-sm6115.c +++ b/drivers/clk/qcom/gcc-sm6115.c @@ -120,7 +120,7 @@ static const struct alpha_pll_config gpll10_config = { .vco_mask = GENMASK(21, 20), .main_output_mask = BIT(0), .config_ctl_val = 0x4001055b, - .test_ctl_hi1_val = 0x1, + .test_ctl_hi_val = 0x1, .test_ctl_hi_mask = 0x1, }; @@ -173,7 +173,7 @@ static const struct alpha_pll_config gpll11_config = { .vco_val = 0x2 << 20, .vco_mask = GENMASK(21, 20), .config_ctl_val = 0x4001055b, - .test_ctl_hi1_val = 0x1, + .test_ctl_hi_val = 0x1, .test_ctl_hi_mask = 0x1, }; @@ -367,7 +367,7 @@ static const struct alpha_pll_config gpll8_config = { .post_div_val = 0x1 << 8, .post_div_mask = GENMASK(11, 8), .config_ctl_val = 0x4001055b, - .test_ctl_hi1_val = 0x1, + .test_ctl_hi_val = 0x1, .test_ctl_hi_mask = 0x1, }; diff --git a/drivers/clk/qcom/gdsc.c b/drivers/clk/qcom/gdsc.c index ee5f86ca50cb77..b99d70149af34e 100644 --- a/drivers/clk/qcom/gdsc.c +++ b/drivers/clk/qcom/gdsc.c @@ -104,14 +104,21 @@ static int gdsc_hwctrl(struct gdsc *sc, bool en) static int gdsc_poll_status(struct gdsc *sc, enum gdsc_status status) { ktime_t start; + int ret; start = ktime_get(); do { - if (gdsc_check_status(sc, status)) + ret = gdsc_check_status(sc, status); + if (ret < 0) + return ret; + if (ret) return 0; } while (ktime_us_delta(ktime_get(), start) < STATUS_POLL_TIMEOUT_US); - if (gdsc_check_status(sc, status)) + ret = gdsc_check_status(sc, status); + if (ret < 0) + return ret; + if (ret) return 0; return -ETIMEDOUT; @@ -493,7 +500,9 @@ static int gdsc_init(struct gdsc *sc) } else if (sc->flags & ALWAYS_ON) { /* If ALWAYS_ON GDSCs are not ON, turn them ON */ - gdsc_enable(&sc->pd); + ret = gdsc_enable(&sc->pd); + if (ret) + return ret; on = true; } @@ -669,10 +678,18 @@ err_pm_subdomain_remove: void gdsc_unregister(struct gdsc_desc *desc) { struct device *dev = desc->dev; + struct gdsc **scs = desc->scs; size_t num = desc->num; + int i; - gdsc_pm_subdomain_remove(desc, num); of_genpd_del_provider(dev->of_node); + gdsc_pm_subdomain_remove(desc, num); + + for (i = 0; i < num; i++) { + if (!scs[i]) + continue; + pm_genpd_remove(&scs[i]->pd); + } } /* @@ -708,3 +725,25 @@ int gdsc_gx_do_nothing_enable(struct generic_pm_domain *domain) return ret; } EXPORT_SYMBOL_GPL(gdsc_gx_do_nothing_enable); + +/* + * GX GDSC is a special power domain. Normally, its disable sequence + * is managed by the GMU firmware, and high level OS must not attempt + * to disable it. The only exception is during GMU recovery, where the + * GMU driver can set GenPD’s synced_poweroff flag to allow explicitly + * disable GX GDSC in hardware. + */ +int gdsc_gx_disable(struct generic_pm_domain *domain) +{ + struct gdsc *sc = domain_to_gdsc(domain); + + if (domain->synced_poweroff) + return gdsc_disable(domain); + + /* Remove parent-supply placed in enable */ + if (sc->rsupply) + return regulator_disable(sc->rsupply); + + return 0; +} +EXPORT_SYMBOL_GPL(gdsc_gx_disable); diff --git a/drivers/clk/qcom/gdsc.h b/drivers/clk/qcom/gdsc.h index 92ff6bcce7b1c1..2f9665b664e606 100644 --- a/drivers/clk/qcom/gdsc.h +++ b/drivers/clk/qcom/gdsc.h @@ -93,6 +93,7 @@ int gdsc_register(struct gdsc_desc *desc, struct reset_controller_dev *, struct regmap *); void gdsc_unregister(struct gdsc_desc *desc); int gdsc_gx_do_nothing_enable(struct generic_pm_domain *domain); +int gdsc_gx_disable(struct generic_pm_domain *domain); #else static inline int gdsc_register(struct gdsc_desc *desc, struct reset_controller_dev *rcdev, diff --git a/drivers/clk/qcom/gpucc-qcm2290.c b/drivers/clk/qcom/gpucc-qcm2290.c index 66dea9d2a0e519..fc33d82bcfb684 100644 --- a/drivers/clk/qcom/gpucc-qcm2290.c +++ b/drivers/clk/qcom/gpucc-qcm2290.c @@ -2,13 +2,12 @@ /* * Copyright (c) 2020, The Linux Foundation. All rights reserved. * Copyright (c) 2024, Linaro Limited + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include #include #include -#include -#include #include #include @@ -20,6 +19,7 @@ #include "clk-regmap-divider.h" #include "clk-regmap-mux.h" #include "clk-regmap-phy-mux.h" +#include "common.h" #include "gdsc.h" #include "reset.h" @@ -56,6 +56,7 @@ static const struct alpha_pll_config gpu_cc_pll0_config = { static struct clk_alpha_pll gpu_cc_pll0 = { .offset = 0x0, + .config = &gpu_cc_pll0_config, .vco_table = huayra_vco, .num_vco = ARRAY_SIZE(huayra_vco), .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_HUAYRA_2290], @@ -143,21 +144,7 @@ static struct clk_rcg2 gpu_cc_gx_gfx3d_clk_src = { .parent_data = gpu_cc_parent_data_1, .num_parents = ARRAY_SIZE(gpu_cc_parent_data_1), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, - }, -}; - -static struct clk_branch gpu_cc_ahb_clk = { - .halt_reg = 0x1078, - .halt_check = BRANCH_HALT_DELAY, - .clkr = { - .enable_reg = 0x1078, - .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ - .name = "gpu_cc_ahb_clk", - .flags = CLK_IS_CRITICAL, - .ops = &clk_branch2_ops, - }, + .ops = &clk_rcg2_shared_ops, }, }; @@ -296,11 +283,14 @@ static struct clk_branch gpu_cc_hlos1_vote_gpu_smmu_clk = { static struct gdsc gpu_cx_gdsc = { .gdscr = 0x106c, .gds_hw_ctrl = 0x1540, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0x2, .pd = { .name = "gpu_cx_gdsc", }, .pwrsts = PWRSTS_OFF_ON, - .flags = VOTABLE, + .flags = RETAIN_FF_ENABLE | VOTABLE, }; static struct gdsc gpu_gx_gdsc = { @@ -308,16 +298,18 @@ static struct gdsc gpu_gx_gdsc = { .clamp_io_ctrl = 0x1508, .resets = (unsigned int []){ GPU_GX_BCR }, .reset_count = 1, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0x2, .pd = { .name = "gpu_gx_gdsc", }, .parent = &gpu_cx_gdsc.pd, .pwrsts = PWRSTS_OFF_ON, - .flags = CLAMP_IO | AON_RESET | SW_RESET, + .flags = RETAIN_FF_ENABLE | POLL_CFG_GDSCR | CLAMP_IO | AON_RESET | SW_RESET, }; static struct clk_regmap *gpu_cc_qcm2290_clocks[] = { - [GPU_CC_AHB_CLK] = &gpu_cc_ahb_clk.clkr, [GPU_CC_CRC_AHB_CLK] = &gpu_cc_crc_ahb_clk.clkr, [GPU_CC_CX_GFX3D_CLK] = &gpu_cc_cx_gfx3d_clk.clkr, [GPU_CC_CX_GMU_CLK] = &gpu_cc_cx_gmu_clk.clkr, @@ -341,6 +333,15 @@ static struct gdsc *gpu_cc_qcm2290_gdscs[] = { [GPU_GX_GDSC] = &gpu_gx_gdsc, }; +static struct clk_alpha_pll *gpu_cc_qcm2290_plls[] = { + &gpu_cc_pll0, +}; + +static const u32 gpu_cc_qcm2290_critical_cbcrs[] = { + 0x1078, /* GPU_CC_AHB_CLK */ + 0x1060, /* GPU_CC_GX_CXO_CLK */ +}; + static const struct regmap_config gpu_cc_qcm2290_regmap_config = { .reg_bits = 32, .reg_stride = 4, @@ -349,6 +350,12 @@ static const struct regmap_config gpu_cc_qcm2290_regmap_config = { .fast_io = true, }; +static const struct qcom_cc_driver_data gpu_cc_qcm2290_driver_data = { + .alpha_plls = gpu_cc_qcm2290_plls, + .num_alpha_plls = ARRAY_SIZE(gpu_cc_qcm2290_plls), + .clk_cbcrs = gpu_cc_qcm2290_critical_cbcrs, + .num_clk_cbcrs = ARRAY_SIZE(gpu_cc_qcm2290_critical_cbcrs), +}; static const struct qcom_cc_desc gpu_cc_qcm2290_desc = { .config = &gpu_cc_qcm2290_regmap_config, @@ -358,6 +365,8 @@ static const struct qcom_cc_desc gpu_cc_qcm2290_desc = { .num_resets = ARRAY_SIZE(gpu_cc_qcm2290_resets), .gdscs = gpu_cc_qcm2290_gdscs, .num_gdscs = ARRAY_SIZE(gpu_cc_qcm2290_gdscs), + .use_rpm = true, + .driver_data = &gpu_cc_qcm2290_driver_data, }; static const struct of_device_id gpu_cc_qcm2290_match_table[] = { @@ -368,45 +377,7 @@ MODULE_DEVICE_TABLE(of, gpu_cc_qcm2290_match_table); static int gpu_cc_qcm2290_probe(struct platform_device *pdev) { - struct regmap *regmap; - int ret; - - regmap = qcom_cc_map(pdev, &gpu_cc_qcm2290_desc); - if (IS_ERR(regmap)) - return PTR_ERR(regmap); - - ret = devm_pm_runtime_enable(&pdev->dev); - if (ret) - return ret; - - ret = devm_pm_clk_create(&pdev->dev); - if (ret) - return ret; - - ret = pm_clk_add(&pdev->dev, NULL); - if (ret < 0) { - dev_err(&pdev->dev, "failed to acquire ahb clock\n"); - return ret; - } - - ret = pm_runtime_resume_and_get(&pdev->dev); - if (ret) - return ret; - - clk_huayra_2290_pll_configure(&gpu_cc_pll0, regmap, &gpu_cc_pll0_config); - - regmap_update_bits(regmap, 0x1060, BIT(0), BIT(0)); /* GPU_CC_GX_CXO_CLK */ - - ret = qcom_cc_really_probe(&pdev->dev, &gpu_cc_qcm2290_desc, regmap); - if (ret) { - dev_err(&pdev->dev, "Failed to register display clock controller\n"); - goto out_pm_runtime_put; - } - -out_pm_runtime_put: - pm_runtime_put_sync(&pdev->dev); - - return 0; + return qcom_cc_probe(pdev, &gpu_cc_qcm2290_desc); } static struct platform_driver gpu_cc_qcm2290_driver = { diff --git a/drivers/clk/qcom/gxclkctl-kaanapali.c b/drivers/clk/qcom/gxclkctl-kaanapali.c index 10c1a8976c56c3..a03da61489b4d7 100644 --- a/drivers/clk/qcom/gxclkctl-kaanapali.c +++ b/drivers/clk/qcom/gxclkctl-kaanapali.c @@ -25,6 +25,7 @@ static struct gdsc gx_clkctl_gx_gdsc = { .pd = { .name = "gx_clkctl_gx_gdsc", .power_on = gdsc_gx_do_nothing_enable, + .power_off = gdsc_gx_disable, }, .pwrsts = PWRSTS_OFF_ON, .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, diff --git a/drivers/clk/qcom/negcc-nord.c b/drivers/clk/qcom/negcc-nord.c index 355850a875acb0..0db284edc4e41e 100644 --- a/drivers/clk/qcom/negcc-nord.c +++ b/drivers/clk/qcom/negcc-nord.c @@ -16,7 +16,6 @@ #include "clk-rcg.h" #include "clk-regmap.h" #include "clk-regmap-divider.h" -#include "clk-regmap-mux.h" #include "clk-regmap-phy-mux.h" #include "common.h" #include "gdsc.h" @@ -41,8 +40,6 @@ enum { P_UFS_PHY_RX_SYMBOL_0_CLK, P_UFS_PHY_RX_SYMBOL_1_CLK, P_UFS_PHY_TX_SYMBOL_0_CLK, - P_USB3_PHY_SEC_WRAPPER_NE_GCC_USB31_PIPE_CLK, - P_USB3_PHY_WRAPPER_NE_GCC_USB31_PIPE_CLK, }; static struct clk_alpha_pll ne_gcc_gpll0 = { @@ -165,26 +162,6 @@ static const struct clk_parent_data ne_gcc_parent_data_5[] = { { .index = DT_BI_TCXO }, }; -static const struct parent_map ne_gcc_parent_map_6[] = { - { P_USB3_PHY_WRAPPER_NE_GCC_USB31_PIPE_CLK, 0 }, - { P_BI_TCXO, 2 }, -}; - -static const struct clk_parent_data ne_gcc_parent_data_6[] = { - { .index = DT_USB3_PHY_WRAPPER_NE_GCC_USB31_PIPE_CLK }, - { .index = DT_BI_TCXO }, -}; - -static const struct parent_map ne_gcc_parent_map_7[] = { - { P_USB3_PHY_SEC_WRAPPER_NE_GCC_USB31_PIPE_CLK, 0 }, - { P_BI_TCXO, 2 }, -}; - -static const struct clk_parent_data ne_gcc_parent_data_7[] = { - { .index = DT_USB3_PHY_SEC_WRAPPER_NE_GCC_USB31_PIPE_CLK }, - { .index = DT_BI_TCXO }, -}; - static struct clk_regmap_phy_mux ne_gcc_ufs_phy_rx_symbol_0_clk_src = { .reg = 0x33068, .clkr = { @@ -227,32 +204,30 @@ static struct clk_regmap_phy_mux ne_gcc_ufs_phy_tx_symbol_0_clk_src = { }, }; -static struct clk_regmap_mux ne_gcc_usb3_prim_phy_pipe_clk_src = { +static struct clk_regmap_phy_mux ne_gcc_usb3_prim_phy_pipe_clk_src = { .reg = 0x2a078, - .shift = 0, - .width = 2, - .parent_map = ne_gcc_parent_map_6, .clkr = { .hw.init = &(const struct clk_init_data) { .name = "ne_gcc_usb3_prim_phy_pipe_clk_src", - .parent_data = ne_gcc_parent_data_6, - .num_parents = ARRAY_SIZE(ne_gcc_parent_data_6), - .ops = &clk_regmap_mux_closest_ops, + .parent_data = &(const struct clk_parent_data){ + .index = DT_USB3_PHY_WRAPPER_NE_GCC_USB31_PIPE_CLK, + }, + .num_parents = 1, + .ops = &clk_regmap_phy_mux_ops, }, }, }; -static struct clk_regmap_mux ne_gcc_usb3_sec_phy_pipe_clk_src = { +static struct clk_regmap_phy_mux ne_gcc_usb3_sec_phy_pipe_clk_src = { .reg = 0x2c078, - .shift = 0, - .width = 2, - .parent_map = ne_gcc_parent_map_7, .clkr = { .hw.init = &(const struct clk_init_data) { .name = "ne_gcc_usb3_sec_phy_pipe_clk_src", - .parent_data = ne_gcc_parent_data_7, - .num_parents = ARRAY_SIZE(ne_gcc_parent_data_7), - .ops = &clk_regmap_mux_closest_ops, + .parent_data = &(const struct clk_parent_data){ + .index = DT_USB3_PHY_SEC_WRAPPER_NE_GCC_USB31_PIPE_CLK, + }, + .num_parents = 1, + .ops = &clk_regmap_phy_mux_ops, }, }, }; @@ -951,21 +926,6 @@ static struct clk_branch ne_gcc_gp2_clk = { }, }; -static struct clk_branch ne_gcc_gpu_2_cfg_clk = { - .halt_reg = 0x34004, - .halt_check = BRANCH_HALT_VOTED, - .hwcg_reg = 0x34004, - .hwcg_bit = 1, - .clkr = { - .enable_reg = 0x34004, - .enable_mask = BIT(0), - .hw.init = &(const struct clk_init_data) { - .name = "ne_gcc_gpu_2_cfg_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch ne_gcc_gpu_2_gpll0_clk_src = { .halt_check = BRANCH_HALT_DELAY, .clkr = { @@ -1641,7 +1601,7 @@ static struct clk_branch ne_gcc_usb3_prim_phy_com_aux_clk = { static struct clk_branch ne_gcc_usb3_prim_phy_pipe_clk = { .halt_reg = 0x2a074, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .hwcg_reg = 0x2a074, .hwcg_bit = 1, .clkr = { @@ -1697,7 +1657,7 @@ static struct clk_branch ne_gcc_usb3_sec_phy_com_aux_clk = { static struct clk_branch ne_gcc_usb3_sec_phy_pipe_clk = { .halt_reg = 0x2c074, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .hwcg_reg = 0x2c074, .hwcg_bit = 1, .clkr = { @@ -1816,7 +1776,6 @@ static struct clk_regmap *ne_gcc_nord_clocks[] = { [NE_GCC_GPLL0] = &ne_gcc_gpll0.clkr, [NE_GCC_GPLL0_OUT_EVEN] = &ne_gcc_gpll0_out_even.clkr, [NE_GCC_GPLL2] = &ne_gcc_gpll2.clkr, - [NE_GCC_GPU_2_CFG_CLK] = &ne_gcc_gpu_2_cfg_clk.clkr, [NE_GCC_GPU_2_GPLL0_CLK_SRC] = &ne_gcc_gpu_2_gpll0_clk_src.clkr, [NE_GCC_GPU_2_GPLL0_DIV_CLK_SRC] = &ne_gcc_gpu_2_gpll0_div_clk_src.clkr, [NE_GCC_GPU_2_HSCNOC_GFX_CLK] = &ne_gcc_gpu_2_hscnoc_gfx_clk.clkr, @@ -1945,10 +1904,16 @@ static void clk_nord_regs_configure(struct device *dev, struct regmap *regmap) qcom_branch_set_force_mem_core(regmap, ne_gcc_ufs_phy_axi_clk, true); } +static const u32 ne_gcc_nord_critical_cbcrs[] = { + 0x34004, /* NE_GCC_GPU_2_CFG_CLK */ +}; + static const struct qcom_cc_driver_data ne_gcc_nord_driver_data = { .dfs_rcgs = ne_gcc_nord_dfs_clocks, .num_dfs_rcgs = ARRAY_SIZE(ne_gcc_nord_dfs_clocks), .clk_regs_configure = clk_nord_regs_configure, + .clk_cbcrs = ne_gcc_nord_critical_cbcrs, + .num_clk_cbcrs = ARRAY_SIZE(ne_gcc_nord_critical_cbcrs), }; static const struct qcom_cc_desc ne_gcc_nord_desc = { diff --git a/drivers/clk/rockchip/clk-pll.c b/drivers/clk/rockchip/clk-pll.c index 6b853800cb6bc6..706ca4b344d399 100644 --- a/drivers/clk/rockchip/clk-pll.c +++ b/drivers/clk/rockchip/clk-pll.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include "clk.h" @@ -900,6 +901,14 @@ static void rockchip_rk3588_pll_get_params(struct rockchip_clk_pll *pll, rate->k = ((pllcon >> RK3588_PLLCON2_K_SHIFT) & RK3588_PLLCON2_K_MASK); } +/* + * 2250 MHz <= Fvco <= 4500 MHz + * For Fvco > 3 GHz: period jitter +-1% frac PLL, +-0.75% int PLL + * For Fvco < 3 GHz: period jitter +-2% frac PLL, +-1.50% int PLL + * Fvco = ((m + k / 65536) * Fin) / p + * Fout = ((m + k / 65536) * Fin) / (p * 2^s) + * -32768 <= k <= 32767 (only available in frac PLLs, not int PLLs) + */ static unsigned long rockchip_rk3588_pll_recalc_rate(struct clk_hw *hw, unsigned long prate) { struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); @@ -913,11 +922,10 @@ static unsigned long rockchip_rk3588_pll_recalc_rate(struct clk_hw *hw, unsigned if (cur.k) { /* fractional mode */ - u64 frac_rate64 = prate * cur.k; + s64 frac_rate64 = (s64)prate * cur.k; - postdiv = cur.p * 65535; - do_div(frac_rate64, postdiv); - rate64 += frac_rate64; + postdiv = cur.p * 65536; + rate64 += div_s64(frac_rate64, postdiv); } rate64 = rate64 >> cur.s; diff --git a/drivers/clk/rockchip/clk-rk3576.c b/drivers/clk/rockchip/clk-rk3576.c index 2557358e0b9d87..2cdd667f400410 100644 --- a/drivers/clk/rockchip/clk-rk3576.c +++ b/drivers/clk/rockchip/clk-rk3576.c @@ -79,13 +79,13 @@ static struct rockchip_pll_rate_table rk3576_pll_rates[] = { RK3588_PLL_RATE(1008000000, 2, 336, 2, 0), RK3588_PLL_RATE(1000000000, 3, 500, 2, 0), RK3588_PLL_RATE(983040000, 4, 655, 2, 23592), - RK3588_PLL_RATE(955520000, 3, 477, 2, 49806), + RK3588_PLL_RATE(955520000, 3, 478, 2, -15730), RK3588_PLL_RATE(903168000, 6, 903, 2, 11009), RK3588_PLL_RATE(900000000, 2, 300, 2, 0), RK3588_PLL_RATE(816000000, 2, 272, 2, 0), RK3588_PLL_RATE(786432000, 2, 262, 2, 9437), RK3588_PLL_RATE(786000000, 1, 131, 2, 0), - RK3588_PLL_RATE(785560000, 3, 392, 2, 51117), + RK3588_PLL_RATE(785560000, 3, 393, 2, -14419), RK3588_PLL_RATE(722534400, 8, 963, 2, 24850), RK3588_PLL_RATE(600000000, 2, 200, 2, 0), RK3588_PLL_RATE(594000000, 2, 198, 2, 0), @@ -315,6 +315,7 @@ PNAME(mux_100m_24m_lclk0_p) = { "clk_cpll_div10", "xin24m", "lclk_asrc_src_0" } PNAME(mux_100m_24m_lclk1_p) = { "clk_cpll_div10", "xin24m", "lclk_asrc_src_1" }; PNAME(mux_150m_100m_50m_24m_p) = { "clk_gpll_div8", "clk_cpll_div10", "clk_cpll_div20", "xin24m" }; PNAME(mux_200m_100m_50m_24m_p) = { "clk_gpll_div6", "clk_cpll_div10", "clk_cpll_div20", "xin24m" }; +PNAME(mux_200m_150m_100m_24m_p) = { "clk_gpll_div6", "clk_gpll_div8", "clk_cpll_div10", "xin24m" }; PNAME(mux_400m_200m_100m_24m_p) = { "clk_gpll_div3", "clk_gpll_div6", "clk_cpll_div10", "xin24m" }; PNAME(mux_500m_250m_100m_24m_p) = { "clk_cpll_div2", "clk_cpll_div4", "clk_cpll_div10", "xin24m" }; PNAME(mux_600m_400m_300m_24m_p) = { "clk_gpll_div2", "clk_gpll_div3", "clk_gpll_div4", "xin24m" }; @@ -706,19 +707,19 @@ static struct rockchip_clk_branch rk3576_clk_branches[] __initdata = { RK3576_CLKGATE_CON(16), 0, GFLAGS), GATE(PCLK_SPI4, "pclk_spi4", "pclk_bus_root", 0, RK3576_CLKGATE_CON(16), 1, GFLAGS), - COMPOSITE_NODIV(CLK_SPI0, "clk_spi0", mux_200m_100m_50m_24m_p, 0, + COMPOSITE_NODIV(CLK_SPI0, "clk_spi0", mux_200m_150m_100m_24m_p, 0, RK3576_CLKSEL_CON(70), 13, 2, MFLAGS, RK3576_CLKGATE_CON(16), 2, GFLAGS), - COMPOSITE_NODIV(CLK_SPI1, "clk_spi1", mux_200m_100m_50m_24m_p, 0, + COMPOSITE_NODIV(CLK_SPI1, "clk_spi1", mux_200m_150m_100m_24m_p, 0, RK3576_CLKSEL_CON(71), 0, 2, MFLAGS, RK3576_CLKGATE_CON(16), 3, GFLAGS), - COMPOSITE_NODIV(CLK_SPI2, "clk_spi2", mux_200m_100m_50m_24m_p, 0, + COMPOSITE_NODIV(CLK_SPI2, "clk_spi2", mux_200m_150m_100m_24m_p, 0, RK3576_CLKSEL_CON(71), 2, 2, MFLAGS, RK3576_CLKGATE_CON(16), 4, GFLAGS), - COMPOSITE_NODIV(CLK_SPI3, "clk_spi3", mux_200m_100m_50m_24m_p, 0, + COMPOSITE_NODIV(CLK_SPI3, "clk_spi3", mux_200m_150m_100m_24m_p, 0, RK3576_CLKSEL_CON(71), 4, 2, MFLAGS, RK3576_CLKGATE_CON(16), 5, GFLAGS), - COMPOSITE_NODIV(CLK_SPI4, "clk_spi4", mux_200m_100m_50m_24m_p, 0, + COMPOSITE_NODIV(CLK_SPI4, "clk_spi4", mux_200m_150m_100m_24m_p, 0, RK3576_CLKSEL_CON(71), 6, 2, MFLAGS, RK3576_CLKGATE_CON(16), 6, GFLAGS), GATE(PCLK_WDT0, "pclk_wdt0", "pclk_bus_root", 0, diff --git a/drivers/clk/rockchip/clk-rk3588.c b/drivers/clk/rockchip/clk-rk3588.c index 86a6870cc2ee35..517e30e249d7b4 100644 --- a/drivers/clk/rockchip/clk-rk3588.c +++ b/drivers/clk/rockchip/clk-rk3588.c @@ -79,14 +79,14 @@ static struct rockchip_pll_rate_table rk3588_pll_rates[] = { RK3588_PLL_RATE(1008000000, 2, 336, 2, 0), RK3588_PLL_RATE(1000000000, 3, 500, 2, 0), RK3588_PLL_RATE(983040000, 4, 655, 2, 23592), - RK3588_PLL_RATE(955520000, 3, 477, 2, 49806), + RK3588_PLL_RATE(955520000, 3, 478, 2, -15730), RK3588_PLL_RATE(903168000, 6, 903, 2, 11009), RK3588_PLL_RATE(900000000, 2, 300, 2, 0), RK3588_PLL_RATE(850000000, 3, 425, 2, 0), RK3588_PLL_RATE(816000000, 2, 272, 2, 0), RK3588_PLL_RATE(786432000, 2, 262, 2, 9437), RK3588_PLL_RATE(786000000, 1, 131, 2, 0), - RK3588_PLL_RATE(785560000, 3, 392, 2, 51117), + RK3588_PLL_RATE(785560000, 3, 393, 2, -14419), RK3588_PLL_RATE(722534400, 8, 963, 2, 24850), RK3588_PLL_RATE(600000000, 2, 200, 2, 0), RK3588_PLL_RATE(594000000, 2, 198, 2, 0), diff --git a/drivers/clk/rockchip/clk.h b/drivers/clk/rockchip/clk.h index 9e3503e2ffc23b..72b36bba315238 100644 --- a/drivers/clk/rockchip/clk.h +++ b/drivers/clk/rockchip/clk.h @@ -635,10 +635,10 @@ struct rockchip_pll_rate_table { }; struct { /* for RK3588 */ - unsigned int m; - unsigned int p; - unsigned int s; - unsigned int k; + unsigned int m; /* main divider, 10 bit unsigned */ + unsigned int p; /* pre-divider, 6 bit unsigned */ + unsigned int s; /* scaler, 3 bit unsigned */ + s16 k; /* fractional part, 16 bit two's complement */ }; }; }; diff --git a/drivers/clk/spacemit/ccu-k3.c b/drivers/clk/spacemit/ccu-k3.c index 764426359540c3..e94a5d2687d571 100644 --- a/drivers/clk/spacemit/ccu-k3.c +++ b/drivers/clk/spacemit/ccu-k3.c @@ -236,11 +236,11 @@ CCU_MUX_DEFINE(i2s3_sysclk_sel, i2s_sysclk_parents, MPMU_I2S_SYSCLK_CTRL, 12, 2, CCU_MUX_DEFINE(i2s4_sysclk_sel, i2s_sysclk_parents, MPMU_I2S_SYSCLK_CTRL, 16, 2, 0); CCU_MUX_DEFINE(i2s5_sysclk_sel, i2s_sysclk_parents, MPMU_I2S_SYSCLK_CTRL, 20, 2, 0); -CCU_DDN_DEFINE(i2s0_sysclk_div, i2s0_sysclk_sel, MPMU_I2S0_SYSCLK, 0, 16, 16, 16, 1, 0); -CCU_DDN_DEFINE(i2s2_sysclk_div, i2s2_sysclk_sel, MPMU_I2S2_SYSCLK, 0, 16, 16, 16, 1, 0); -CCU_DDN_DEFINE(i2s3_sysclk_div, i2s3_sysclk_sel, MPMU_I2S3_SYSCLK, 0, 16, 16, 16, 1, 0); -CCU_DDN_DEFINE(i2s4_sysclk_div, i2s4_sysclk_sel, MPMU_I2S4_SYSCLK, 0, 16, 16, 16, 1, 0); -CCU_DDN_DEFINE(i2s5_sysclk_div, i2s5_sysclk_sel, MPMU_I2S5_SYSCLK, 0, 16, 16, 16, 1, 0); +CCU_DDN_DEFINE(i2s0_sysclk_div, i2s0_sysclk_sel, MPMU_I2S0_SYSCLK, 0, 16, 16, 16, 2, 0); +CCU_DDN_DEFINE(i2s2_sysclk_div, i2s2_sysclk_sel, MPMU_I2S2_SYSCLK, 0, 16, 16, 16, 2, 0); +CCU_DDN_DEFINE(i2s3_sysclk_div, i2s3_sysclk_sel, MPMU_I2S3_SYSCLK, 0, 16, 16, 16, 2, 0); +CCU_DDN_DEFINE(i2s4_sysclk_div, i2s4_sysclk_sel, MPMU_I2S4_SYSCLK, 0, 16, 16, 16, 2, 0); +CCU_DDN_DEFINE(i2s5_sysclk_div, i2s5_sysclk_sel, MPMU_I2S5_SYSCLK, 0, 16, 16, 16, 2, 0); static const struct clk_parent_data i2s2_sysclk_parents[] = { CCU_PARENT_HW(i2s1_sysclk), @@ -926,11 +926,10 @@ CCU_MUX_DIV_GATE_FC_DEFINE(dpu_aclk, dpu_aclk_parents, APMU_LCD_CLK_RES_CTRL5, 1 20, 3, BIT(16), 0); static const struct clk_parent_data ufs_aclk_parents[] = { - CCU_PARENT_HW(pll1_d6_409p6), CCU_PARENT_HW(pll1_d5_491p52), - CCU_PARENT_HW(pll1_d4_614p4), - CCU_PARENT_HW(pll1_d8_307p2), - CCU_PARENT_HW(pll2_d4), + CCU_PARENT_HW(pll1_d6_409p6), + CCU_PARENT_HW(pll2_d6), + CCU_PARENT_HW(pll2_d5), }; CCU_MUX_DIV_GATE_FC_DEFINE(ufs_aclk, ufs_aclk_parents, APMU_UFS_CLK_RES_CTRL, 5, 3, BIT(8), 2, 3, BIT(1), 0); diff --git a/drivers/clk/stm32/clk-stm32mp21.c b/drivers/clk/stm32/clk-stm32mp21.c index c8a37b716bd559..bdb17419908c8f 100644 --- a/drivers/clk/stm32/clk-stm32mp21.c +++ b/drivers/clk/stm32/clk-stm32mp21.c @@ -4,6 +4,7 @@ * Author: Gabriel Fernandez for STMicroelectronics. */ +#include #include #include #include diff --git a/drivers/clk/stm32/clk-stm32mp25.c b/drivers/clk/stm32/clk-stm32mp25.c index 52f0e8a1292626..eb0bc918ecee00 100644 --- a/drivers/clk/stm32/clk-stm32mp25.c +++ b/drivers/clk/stm32/clk-stm32mp25.c @@ -4,6 +4,7 @@ * Author: Gabriel Fernandez for STMicroelectronics. */ +#include #include #include #include diff --git a/drivers/clk/sunxi-ng/ccu_mux.c b/drivers/clk/sunxi-ng/ccu_mux.c index 766f27cff748ed..e56a3005548e3d 100644 --- a/drivers/clk/sunxi-ng/ccu_mux.c +++ b/drivers/clk/sunxi-ng/ccu_mux.c @@ -93,66 +93,65 @@ int ccu_mux_helper_determine_rate(struct ccu_common *common, struct clk_rate_request adj_req = *req; best_parent = clk_hw_get_parent(hw); - best_parent_rate = clk_hw_get_rate(best_parent); - + adj_req.best_parent_rate = clk_hw_get_rate(best_parent); adj_req.best_parent_hw = best_parent; - adj_req.best_parent_rate = ccu_mux_helper_apply_prediv(common, cm, -1, - best_parent_rate); + + /* + * This effectively treats the predivider as a postdivider. + * It stays mathematically correct and ensures whatever + * round() will do stays correct while walking the tree. + * It may query the parent rate too while handling rate + * propagation. + */ + adj_req.rate = ccu_mux_helper_unapply_prediv(common, cm, -1, + req->rate); ret = round(cm, &adj_req, data); if (ret) return ret; - best_rate = adj_req.rate; - /* - * best_parent_rate might have been modified by our clock. - * Unapply the pre-divider if there's one, and give - * the actual frequency the parent needs to run at. + * parent_rate might have been modified by our clock as part + * of the rate propagation mechanism. Same goes below. */ - best_parent_rate = ccu_mux_helper_unapply_prediv(common, cm, -1, - adj_req.best_parent_rate); + best_parent_rate = adj_req.best_parent_rate; + best_rate = ccu_mux_helper_apply_prediv(common, cm, -1, + adj_req.rate); goto out; } for (i = 0; i < clk_hw_get_num_parents(hw); i++) { struct clk_rate_request tmp_req = *req; - unsigned long parent_rate; + unsigned long rate; struct clk_hw *parent; parent = clk_hw_get_parent_by_index(hw, i); if (!parent) continue; - parent_rate = ccu_mux_helper_apply_prediv(common, cm, i, - clk_hw_get_rate(parent)); - tmp_req.best_parent_hw = parent; - tmp_req.best_parent_rate = parent_rate; + tmp_req.best_parent_rate = clk_hw_get_rate(parent); + tmp_req.rate = ccu_mux_helper_unapply_prediv(common, cm, i, + req->rate); ret = round(cm, &tmp_req, data); if (ret) continue; - /* - * parent_rate might have been modified by our clock. - * Unapply the pre-divider if there's one, and give - * the actual frequency the parent needs to run at. - */ - parent_rate = ccu_mux_helper_unapply_prediv(common, cm, i, - tmp_req.best_parent_rate); + rate = ccu_mux_helper_apply_prediv(common, cm, i, + tmp_req.rate); - if (tmp_req.rate == req->rate) { + if (rate == req->rate) { best_parent = parent; - best_parent_rate = parent_rate; - best_rate = tmp_req.rate; + best_parent_rate = tmp_req.best_parent_rate; + best_rate = rate; goto out; } - if (ccu_is_better_rate(common, req->rate, tmp_req.rate, best_rate)) { - best_rate = tmp_req.rate; - best_parent_rate = parent_rate; + if (ccu_is_better_rate(common, req->rate, rate, best_rate)) { + best_rate = rate; + best_parent_rate = tmp_req.best_parent_rate; best_parent = parent; } } diff --git a/drivers/clk/tegra/clk-tegra124-emc.c b/drivers/clk/tegra/clk-tegra124-emc.c index f3b2c96fdcfc24..94ac24ea1e6adc 100644 --- a/drivers/clk/tegra/clk-tegra124-emc.c +++ b/drivers/clk/tegra/clk-tegra124-emc.c @@ -537,6 +537,7 @@ struct clk *tegra124_clk_register_emc(void __iomem *base, struct device_node *np clk = clk_register(NULL, &tegra->hw); if (IS_ERR(clk)) { + of_node_put(tegra->emc_node); kfree(tegra); return clk; } diff --git a/drivers/clk/ti/adpll.c b/drivers/clk/ti/adpll.c index e305fcbac6475b..8885d28face50b 100644 --- a/drivers/clk/ti/adpll.c +++ b/drivers/clk/ti/adpll.c @@ -483,7 +483,7 @@ static const struct clk_ops ti_adpll_ops = { static int ti_adpll_init_dco(struct ti_adpll_data *d) { - struct clk_init_data init; + struct clk_init_data init = {}; struct clk *clock; const char *postfix; int width, err; @@ -576,7 +576,7 @@ static int ti_adpll_init_clkout(struct ti_adpll_data *d, struct clk *clk1) { struct ti_adpll_clkout_data *co; - struct clk_init_data init; + struct clk_init_data init = {}; struct clk_ops *ops; const char *parent_names[2]; const char *child_name; diff --git a/drivers/clk/ti/divider.c b/drivers/clk/ti/divider.c index af923b8cb0ed8f..3b438c2d68aacd 100644 --- a/drivers/clk/ti/divider.c +++ b/drivers/clk/ti/divider.c @@ -311,7 +311,7 @@ static struct clk *_register_divider(struct device_node *node, u32 flags, struct clk_omap_divider *div) { - struct clk_init_data init; + struct clk_init_data init = {}; const char *parent_name; const char *name; diff --git a/drivers/clk/ti/mux.c b/drivers/clk/ti/mux.c index d6a0ccfd81dbec..0fef60e82107a5 100644 --- a/drivers/clk/ti/mux.c +++ b/drivers/clk/ti/mux.c @@ -119,14 +119,14 @@ const struct clk_ops ti_clk_mux_ops = { }; static struct clk *_register_mux(struct device_node *node, const char *name, - const char * const *parent_names, + const struct clk_parent_data *parent_data, u8 num_parents, unsigned long flags, struct clk_omap_reg *reg, u8 shift, u32 mask, s8 latch, u8 clk_mux_flags, u32 *table) { + struct clk_init_data init = {}; struct clk_omap_mux *mux; struct clk *clk; - struct clk_init_data init; /* allocate the mux */ mux = kzalloc_obj(*mux); @@ -136,7 +136,7 @@ static struct clk *_register_mux(struct device_node *node, const char *name, init.name = name; init.ops = &ti_clk_mux_ops; init.flags = flags; - init.parent_names = parent_names; + init.parent_data = parent_data; init.num_parents = num_parents; /* struct clk_mux assignments */ @@ -167,24 +167,26 @@ static void of_mux_clk_setup(struct device_node *node) struct clk *clk; struct clk_omap_reg reg; unsigned int num_parents; - const char **parent_names; + struct clk_parent_data *parent_data; const char *name; u8 clk_mux_flags = 0; u32 mask = 0; u32 shift = 0; s32 latch = -EINVAL; u32 flags = CLK_SET_RATE_NO_REPARENT; + int i; num_parents = of_clk_get_parent_count(node); if (num_parents < 2) { pr_err("mux-clock %pOFn must have parents\n", node); return; } - parent_names = kcalloc(num_parents, sizeof(char *), GFP_KERNEL); - if (!parent_names) - goto cleanup; + parent_data = kcalloc(num_parents, sizeof(*parent_data), GFP_KERNEL); + if (!parent_data) + return; - of_clk_parent_fill(node, parent_names, num_parents); + for (i = 0; i < num_parents; i++) + parent_data[i].index = i; if (ti_clk_get_reg_addr(node, 0, ®)) goto cleanup; @@ -207,7 +209,7 @@ static void of_mux_clk_setup(struct device_node *node) mask = (1 << fls(mask)) - 1; name = ti_dt_clk_name(node); - clk = _register_mux(node, name, parent_names, num_parents, + clk = _register_mux(node, name, parent_data, num_parents, flags, ®, shift, mask, latch, clk_mux_flags, NULL); @@ -215,7 +217,7 @@ static void of_mux_clk_setup(struct device_node *node) of_clk_add_provider(node, of_clk_src_simple_get, clk); cleanup: - kfree(parent_names); + kfree(parent_data); } CLK_OF_DECLARE(mux_clk, "ti,mux-clock", of_mux_clk_setup); diff --git a/drivers/clk/visconti/clkc.c b/drivers/clk/visconti/clkc.c index d0b193b5d0b35f..4018d1298880df 100644 --- a/drivers/clk/visconti/clkc.c +++ b/drivers/clk/visconti/clkc.c @@ -81,9 +81,9 @@ static struct clk_hw *visconti_clk_register_gate(struct device *dev, u8 rs_idx, spinlock_t *lock) { + struct clk_init_data init = {}; struct visconti_clk_gate *gate; struct clk_parent_data *pdata; - struct clk_init_data init; struct clk_hw *hw; int ret; diff --git a/drivers/clk/x86/clk-pmc-atom.c b/drivers/clk/x86/clk-pmc-atom.c index 99291ba65da73d..08c83e0abc41d9 100644 --- a/drivers/clk/x86/clk-pmc-atom.c +++ b/drivers/clk/x86/clk-pmc-atom.c @@ -160,6 +160,9 @@ static struct clk_plt *plt_clk_register(struct platform_device *pdev, int id, return ERR_PTR(-ENOMEM); init.name = kasprintf(GFP_KERNEL, "%s_%d", PLT_CLK_NAME_BASE, id); + if (!init.name) + return ERR_PTR(-ENOMEM); + init.ops = &plt_clk_ops; init.flags = 0; init.parent_names = parent_names; diff --git a/drivers/clocksource/clps711x-timer.c b/drivers/clocksource/clps711x-timer.c index bb0a44adaf28ac..63ae3a691b1483 100644 --- a/drivers/clocksource/clps711x-timer.c +++ b/drivers/clocksource/clps711x-timer.c @@ -94,7 +94,7 @@ static int __init clps711x_timer_init(struct device_node *np) switch (of_alias_get_id(np, "timer")) { case CLPS711X_CLKSRC_CLOCKSOURCE: clps711x_clksrc_init(clock, base); - break; + return 0; case CLPS711X_CLKSRC_CLOCKEVENT: ret = _clps711x_clkevt_init(clock, base, irq); break; diff --git a/drivers/clocksource/samsung_pwm_timer.c b/drivers/clocksource/samsung_pwm_timer.c index b9561e3f196c42..0544124cf5cead 100644 --- a/drivers/clocksource/samsung_pwm_timer.c +++ b/drivers/clocksource/samsung_pwm_timer.c @@ -56,7 +56,7 @@ #define TCON_AUTORELOAD(chan) \ ((chan < 5) ? _TCON_AUTORELOAD(chan) : _TCON_AUTORELOAD4(chan)) -DEFINE_SPINLOCK(samsung_pwm_lock); +DEFINE_RAW_SPINLOCK(samsung_pwm_lock); EXPORT_SYMBOL(samsung_pwm_lock); struct samsung_pwm_clocksource { @@ -87,14 +87,14 @@ static void samsung_timer_set_prescale(unsigned int channel, u16 prescale) if (channel >= 2) shift = TCFG0_PRESCALER1_SHIFT; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); reg = readl(pwm.base + REG_TCFG0); reg &= ~(TCFG0_PRESCALER_MASK << shift); reg |= (prescale - 1) << shift; writel(reg, pwm.base + REG_TCFG0); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); } static void samsung_timer_set_divisor(unsigned int channel, u8 divisor) @@ -106,14 +106,14 @@ static void samsung_timer_set_divisor(unsigned int channel, u8 divisor) bits = (fls(divisor) - 1) - pwm.variant.div_base; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); reg = readl(pwm.base + REG_TCFG1); reg &= ~(TCFG1_MUX_MASK << shift); reg |= bits << shift; writel(reg, pwm.base + REG_TCFG1); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); } static void samsung_time_stop(unsigned int channel) @@ -124,13 +124,13 @@ static void samsung_time_stop(unsigned int channel) if (channel > 0) ++channel; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); tcon = readl_relaxed(pwm.base + REG_TCON); tcon &= ~TCON_START(channel); writel_relaxed(tcon, pwm.base + REG_TCON); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); } static void samsung_time_setup(unsigned int channel, unsigned long tcnt) @@ -142,7 +142,7 @@ static void samsung_time_setup(unsigned int channel, unsigned long tcnt) if (tcon_chan > 0) ++tcon_chan; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); tcon = readl_relaxed(pwm.base + REG_TCON); @@ -153,7 +153,7 @@ static void samsung_time_setup(unsigned int channel, unsigned long tcnt) writel_relaxed(tcnt, pwm.base + REG_TCMPB(channel)); writel_relaxed(tcon, pwm.base + REG_TCON); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); } static void samsung_time_start(unsigned int channel, bool periodic) @@ -164,7 +164,7 @@ static void samsung_time_start(unsigned int channel, bool periodic) if (channel > 0) ++channel; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); tcon = readl_relaxed(pwm.base + REG_TCON); @@ -178,7 +178,7 @@ static void samsung_time_start(unsigned int channel, bool periodic) writel_relaxed(tcon, pwm.base + REG_TCON); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); } static int samsung_set_next_event(unsigned long cycles, diff --git a/drivers/clocksource/timer-armada-370-xp.c b/drivers/clocksource/timer-armada-370-xp.c index a405a084cf72dd..b5a984aa1cbbb3 100644 --- a/drivers/clocksource/timer-armada-370-xp.c +++ b/drivers/clocksource/timer-armada-370-xp.c @@ -349,7 +349,11 @@ static int __init armada_xp_timer_init(struct device_node *np) timer_clk = clk_get_rate(clk); - return armada_370_xp_timer_common_init(np); + ret = armada_370_xp_timer_common_init(np); + if (ret) + clk_disable_unprepare(clk); + + return ret; } TIMER_OF_DECLARE(armada_xp, "marvell,armada-xp-timer", armada_xp_timer_init); @@ -387,7 +391,11 @@ static int __init armada_375_timer_init(struct device_node *np) timer25Mhz = false; } - return armada_370_xp_timer_common_init(np); + ret = armada_370_xp_timer_common_init(np); + if (ret) + clk_disable_unprepare(clk); + + return ret; } TIMER_OF_DECLARE(armada_375, "marvell,armada-375-timer", armada_375_timer_init); @@ -410,7 +418,11 @@ static int __init armada_370_timer_init(struct device_node *np) timer_clk = clk_get_rate(clk) / TIMER_DIVIDER; timer25Mhz = false; - return armada_370_xp_timer_common_init(np); + ret = armada_370_xp_timer_common_init(np); + if (ret) + clk_disable_unprepare(clk); + + return ret; } TIMER_OF_DECLARE(armada_370, "marvell,armada-370-timer", armada_370_timer_init); diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index 735b29f76438af..2142838ad6cc49 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -560,6 +560,11 @@ out: static int __init amd_pstate_ut_init(void) { u32 i = 0, arr_size = ARRAY_SIZE(amd_pstate_ut_cases); + enum amd_pstate_mode mode = amd_pstate_get_status(); + + /* don't test if no running amd-pstate driver */ + if (mode == AMD_PSTATE_UNDEFINED || mode == AMD_PSTATE_DISABLE) + return -EOPNOTSUPP; for (i = 0; i < arr_size; i++) { int ret; diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 3a6b4b224a66d6..80d99ba1902c9c 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -199,7 +199,7 @@ static inline int get_mode_idx_from_str(const char *str, size_t size) static DEFINE_MUTEX(amd_pstate_driver_lock); -static u8 msr_get_epp(struct amd_cpudata *cpudata) +static int msr_get_epp(struct amd_cpudata *cpudata) { u64 value; int ret; @@ -215,12 +215,12 @@ static u8 msr_get_epp(struct amd_cpudata *cpudata) DEFINE_STATIC_CALL(amd_pstate_get_epp, msr_get_epp); -static inline s16 amd_pstate_get_epp(struct amd_cpudata *cpudata) +static inline int amd_pstate_get_epp(struct amd_cpudata *cpudata) { return static_call(amd_pstate_get_epp)(cpudata); } -static u8 shmem_get_epp(struct amd_cpudata *cpudata) +static int shmem_get_epp(struct amd_cpudata *cpudata) { u64 epp; int ret; @@ -526,9 +526,6 @@ static int shmem_init_perf(struct amd_cpudata *cpudata) WRITE_ONCE(cpudata->perf, perf); WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf); - if (cppc_state == AMD_PSTATE_ACTIVE) - return 0; - ret = cppc_get_auto_sel(cpudata->cpu, &auto_sel); if (ret) { pr_warn("failed to get auto_sel, ret: %d\n", ret); @@ -702,9 +699,12 @@ static void amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) WRITE_ONCE(cpudata->max_limit_freq, policy->max); if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { + u8 min_limit_perf = perf.bios_min_perf ?: perf.nominal_perf; + u32 min_limit_freq; + /* - * For performance policy, set MinPerf to nominal_perf rather than - * highest_perf or lowest_nonlinear_perf. + * For performance policy, set MinPerf to nominal_perf / bios_min_perf + * rather than highest_perf or lowest_nonlinear_perf. * * Per commit 0c411b39e4f4c, using highest_perf was observed * to cause frequency throttling on power-limited platforms, leading to @@ -712,11 +712,18 @@ static void amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) * performance too much for HPC workloads requiring high frequency * operation and minimal wakeup latency from idle states. * - * nominal_perf therefore provides a balance by avoiding throttling - * while still maintaining enough performance for HPC workloads. + * nominal_perf therefore provides a balanced default by avoiding + * throttling while still maintaining enough performance for HPC + * workloads when bios_min_perf is not available. + * + * When bios_min_perf is available, users have profiled their workloads + * to understand the best idling frequency. Use that instead. */ - perf.min_limit_perf = min(perf.nominal_perf, perf.max_limit_perf); - WRITE_ONCE(cpudata->min_limit_freq, min(cpudata->nominal_freq, cpudata->max_limit_freq)); + min_limit_perf = min(min_limit_perf, perf.max_limit_perf); + min_limit_freq = perf_to_freq(perf, cpudata->nominal_freq, min_limit_perf); + perf.min_limit_perf = min_limit_perf; + + WRITE_ONCE(cpudata->min_limit_freq, min(min_limit_freq, cpudata->max_limit_freq)); } else { perf.min_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->min); WRITE_ONCE(cpudata->min_limit_freq, policy->min); @@ -1173,6 +1180,9 @@ static int amd_pstate_power_supply_notifier(struct notifier_block *nb, if (cpudata->current_profile != PLATFORM_PROFILE_BALANCED) return 0; + if (!policy) + return NOTIFY_OK; + epp = amd_pstate_get_balanced_epp(policy); ret = amd_pstate_set_epp(policy, epp); @@ -1208,6 +1218,9 @@ static int amd_pstate_profile_set(struct device *dev, struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); int ret; + if (!policy) + return -ENODEV; + switch (profile) { case PLATFORM_PROFILE_LOW_POWER: ret = amd_pstate_set_epp(policy, AMD_CPPC_EPP_POWERSAVE); @@ -1876,6 +1889,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) struct amd_cpudata *cpudata; union perf_cached perf; struct device *dev; + int default_epp; int ret; /* @@ -1924,6 +1938,13 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) policy->boost_supported = READ_ONCE(cpudata->boost_supported); + /* Fetch the firmware programmed default EPP value */ + default_epp = amd_pstate_get_epp(cpudata); + if (default_epp < 0) { + ret = default_epp; + goto free_cpudata1; + } + /* * Set the policy to provide a valid fallback value in case * the default cpufreq governor is neither powersave nor performance. @@ -1931,7 +1952,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) if (amd_pstate_acpi_pm_profile_server() || amd_pstate_acpi_pm_profile_undefined()) { policy->policy = CPUFREQ_POLICY_PERFORMANCE; - cpudata->epp_default_ac = cpudata->epp_default_dc = amd_pstate_get_epp(cpudata); + cpudata->epp_default_ac = cpudata->epp_default_dc = default_epp; cpudata->current_profile = PLATFORM_PROFILE_PERFORMANCE; } else { policy->policy = CPUFREQ_POLICY_POWERSAVE; diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c index e93697d3edfd9b..731f5721ff1ed1 100644 --- a/drivers/cpufreq/imx6q-cpufreq.c +++ b/drivers/cpufreq/imx6q-cpufreq.c @@ -55,7 +55,6 @@ static unsigned int max_freq; static unsigned int transition_latency; static u32 *imx6_soc_volt; -static u32 soc_opp_count; static int imx6q_set_target(struct cpufreq_policy *policy, unsigned int index) { @@ -330,6 +329,7 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev) const struct property *prop; const __be32 *val; u32 nr, i, j; + u32 soc_opp_count = 0; cpu_dev = get_cpu_device(0); if (!cpu_dev) { @@ -400,7 +400,7 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev) } /* Make imx6_soc_volt array's size same as arm opp number */ - imx6_soc_volt = devm_kcalloc(cpu_dev, num, sizeof(*imx6_soc_volt), + imx6_soc_volt = devm_kcalloc(&pdev->dev, num, sizeof(*imx6_soc_volt), GFP_KERNEL); if (imx6_soc_volt == NULL) { ret = -ENOMEM; @@ -485,6 +485,7 @@ soc_opp_out: return 0; free_freq_table: + imx6_soc_volt = NULL; dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); out_free_opp: dev_pm_opp_of_remove_table(cpu_dev); @@ -506,6 +507,7 @@ put_node: static void imx6q_cpufreq_remove(struct platform_device *pdev) { cpufreq_unregister_driver(&imx6q_cpufreq_driver); + imx6_soc_volt = NULL; dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); dev_pm_opp_of_remove_table(cpu_dev); regulator_put(arm_reg); diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 6e984c114d96f8..311087197d7fa2 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -2356,8 +2356,6 @@ static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) if (pstate_funcs.get_vid) pstate_funcs.get_vid(cpu); - - intel_pstate_set_min_pstate(cpu); } /* @@ -3063,6 +3061,7 @@ static int __intel_pstate_cpu_init(struct cpufreq_policy *policy) static int intel_pstate_cpu_init(struct cpufreq_policy *policy) { int ret = __intel_pstate_cpu_init(policy); + struct cpudata *cpu; if (ret) return ret; @@ -3073,11 +3072,11 @@ static int intel_pstate_cpu_init(struct cpufreq_policy *policy) */ policy->policy = CPUFREQ_POLICY_POWERSAVE; - if (hwp_active) { - struct cpudata *cpu = all_cpu_data[policy->cpu]; - + cpu = all_cpu_data[policy->cpu]; + if (hwp_active) cpu->epp_cached = intel_pstate_get_epp(cpu, 0); - } + else + intel_pstate_set_min_pstate(cpu); return 0; } @@ -3301,8 +3300,6 @@ static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy) return ret; policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY; - /* This reflects the intel_pstate_get_cpu_pstates() setting. */ - policy->cur = policy->cpuinfo.min_freq; req = kzalloc_objs(*req, 2); if (!req) { @@ -3323,9 +3320,15 @@ static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy) WRITE_ONCE(cpu->hwp_req_cached, value); cpu->epp_cached = intel_pstate_get_epp(cpu, value); + + intel_cpufreq_hwp_update(cpu, cpu->pstate.min_pstate, + cpu->pstate.max_pstate, + cpu->pstate.min_pstate, false); } else { policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY; + intel_pstate_set_min_pstate(cpu); } + policy->cur = policy->cpuinfo.min_freq; freq = DIV_ROUND_UP(cpu->pstate.turbo_freq * global.min_perf_pct, 100); diff --git a/drivers/cpufreq/spear-cpufreq.c b/drivers/cpufreq/spear-cpufreq.c index 81a0780b2ebf00..ffe5eda82f0b11 100644 --- a/drivers/cpufreq/spear-cpufreq.c +++ b/drivers/cpufreq/spear-cpufreq.c @@ -79,9 +79,9 @@ static int spear1340_set_cpu_rate(struct clk *sys_pclk, unsigned long newfreq) int ret = 0; sys_clk = clk_get_parent(spear_cpufreq.clk); - if (IS_ERR(sys_clk)) { + if (!sys_clk) { pr_err("failed to get cpu's parent (sys) clock\n"); - return PTR_ERR(sys_clk); + return -EINVAL; } /* Set the rate of the source clock before changing the parent */ diff --git a/drivers/crypto/aspeed/aspeed-acry.c b/drivers/crypto/aspeed/aspeed-acry.c index 5993bcba971635..301612556a7691 100644 --- a/drivers/crypto/aspeed/aspeed-acry.c +++ b/drivers/crypto/aspeed/aspeed-acry.c @@ -728,7 +728,7 @@ static int aspeed_acry_probe(struct platform_device *pdev) /* Get irq number and register it */ acry_dev->irq = platform_get_irq(pdev, 0); if (acry_dev->irq < 0) - return -ENXIO; + return acry_dev->irq; rc = devm_request_irq(dev, acry_dev->irq, aspeed_acry_irq, 0, dev_name(dev), acry_dev); diff --git a/drivers/crypto/aspeed/aspeed-hace.c b/drivers/crypto/aspeed/aspeed-hace.c index 3fe644bfe0373e..1f9afa002ae8c6 100644 --- a/drivers/crypto/aspeed/aspeed-hace.c +++ b/drivers/crypto/aspeed/aspeed-hace.c @@ -127,7 +127,7 @@ static int aspeed_hace_probe(struct platform_device *pdev) /* Get irq number and register it */ hace_dev->irq = platform_get_irq(pdev, 0); if (hace_dev->irq < 0) - return -ENXIO; + return hace_dev->irq; rc = devm_request_irq(&pdev->dev, hace_dev->irq, aspeed_hace_irq, 0, dev_name(&pdev->dev), hace_dev); diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c index 4dc34c5bc0f6c4..bd1664ad5c0eac 100644 --- a/drivers/crypto/atmel-ecc.c +++ b/drivers/crypto/atmel-ecc.c @@ -165,6 +165,9 @@ static int atmel_ecdh_compute_shared_secret(struct kpp_request *req) return crypto_kpp_compute_shared_secret(req); } + if (!ctx->public_key) + return -EINVAL; + /* A P-256 public key must contain two 32-byte coordinates */ if (req->src_len != ATMEL_ECC_PUBKEY_SIZE) return -EINVAL; diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c index 4c9af737b33aa2..5eb76245347d87 100644 --- a/drivers/crypto/atmel-sha204a.c +++ b/drivers/crypto/atmel-sha204a.c @@ -31,10 +31,14 @@ static void atmel_sha204a_rng_done(struct atmel_i2c_work_data *work_data, struct atmel_i2c_client_priv *i2c_priv = work_data->ctx; struct hwrng *rng = areq; - if (status) + if (status) { dev_warn_ratelimited(&i2c_priv->client->dev, "i2c transaction failed (%d)\n", status); + kfree(work_data); + atomic_dec(&i2c_priv->tfm_count); + return; + } rng->priv = (unsigned long)work_data; atomic_dec(&i2c_priv->tfm_count); diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index ca473ca198b81f..e7db638c25276e 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -1545,7 +1545,7 @@ static int __sev_platform_init_handle_init_ex_path(struct sev_device *sev) if (sev_init_ex_buffer) return 0; - page = alloc_pages(GFP_KERNEL, get_order(NV_LENGTH)); + page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(NV_LENGTH)); if (!page) { dev_err(sev->dev, "SEV: INIT_EX NV memory allocation failed\n"); return -ENOMEM; @@ -1555,20 +1555,31 @@ static int __sev_platform_init_handle_init_ex_path(struct sev_device *sev) rc = sev_read_init_ex_file(); if (rc) - return rc; + goto err_free; /* If SEV-SNP is initialized, transition to firmware page. */ if (sev->snp_initialized) { unsigned long npages; npages = 1UL << get_order(NV_LENGTH); - if (rmp_mark_pages_firmware(__pa(sev_init_ex_buffer), npages, false)) { + if (rmp_mark_pages_firmware(__pa(sev_init_ex_buffer), npages, true)) { dev_err(sev->dev, "SEV: INIT_EX NV memory page state change failed.\n"); - return -ENOMEM; + rc = -ENOMEM; + /* + * Pages can be in an inconsistent state, don't release them back to the + * system. + */ + goto err_reset; } } return 0; + +err_free: + __free_pages(page, get_order(NV_LENGTH)); +err_reset: + sev_init_ex_buffer = NULL; + return rc; } static int __sev_platform_init_locked(int *error) diff --git a/drivers/crypto/gemini/sl3516-ce-cipher.c b/drivers/crypto/gemini/sl3516-ce-cipher.c index 583010b2d00712..02ec4282333b6f 100644 --- a/drivers/crypto/gemini/sl3516-ce-cipher.c +++ b/drivers/crypto/gemini/sl3516-ce-cipher.c @@ -56,10 +56,6 @@ static bool sl3516_ce_need_fallback(struct skcipher_request *areq) ce->fallback_mod16++; return true; } - if ((sg_dma_len(sg) % 16) != 0) { - ce->fallback_mod16++; - return true; - } if (!IS_ALIGNED(sg->offset, 16)) { ce->fallback_align16++; return true; @@ -72,10 +68,6 @@ static bool sl3516_ce_need_fallback(struct skcipher_request *areq) ce->fallback_mod16++; return true; } - if ((sg_dma_len(sg) % 16) != 0) { - ce->fallback_mod16++; - return true; - } if (!IS_ALIGNED(sg->offset, 16)) { ce->fallback_align16++; return true; diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 77e0e03cbcab98..a7354fd10dc0f6 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -96,7 +96,6 @@ #define IV_FLAGS_OFFSET 0x6 #define IV_CM_OFFSET 0x3 #define IV_LAST_BYTE1 1 -#define IV_LAST_BYTE2 2 #define IV_LAST_BYTE_MASK 0xFF #define IV_CTR_INIT 0x1 #define IV_BYTE_OFFSET 0x8 @@ -1696,7 +1695,7 @@ static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req) struct sec_cipher_req *c_req = &req->c_req; u32 data_size = aead_req->cryptlen; u8 flage = 0; - u8 cm, cl; + u8 cm, cl, i; /* the specification has been checked in aead_iv_demension_check() */ cl = c_req->c_ivin[0] + 1; @@ -1720,15 +1719,16 @@ static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req) * the last 32bit is counter's initial number, * but the nonce uses the first 16bit * the tail 16bit fill with the cipher length + * When CL is 3, the tail 24bit fill with the cipher length. */ if (!c_req->encrypt) data_size = aead_req->cryptlen - authsize; - a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = - data_size & IV_LAST_BYTE_MASK; - data_size >>= IV_BYTE_OFFSET; - a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE2] = + for (i = 1; i <= cl; i++) { + a_req->a_ivin[ctx->c_ctx.ivsize - i] = data_size & IV_LAST_BYTE_MASK; + data_size >>= IV_BYTE_OFFSET; + } } static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req) diff --git a/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c b/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c index 8a8f6c81e010c2..419f88af1031b0 100644 --- a/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c +++ b/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c @@ -1541,7 +1541,7 @@ static int register_aes_algs(struct ocs_aes_dev *aes_dev) /* * If any algorithm fails to register, all preceding algorithms that - * were successfully registered will be automatically unregistered. + * were registered in the same call are automatically unregistered. */ ret = crypto_engine_register_aeads(algs_aead, ARRAY_SIZE(algs_aead)); if (ret) @@ -1549,7 +1549,7 @@ static int register_aes_algs(struct ocs_aes_dev *aes_dev) ret = crypto_engine_register_skciphers(algs, ARRAY_SIZE(algs)); if (ret) - crypto_engine_unregister_aeads(algs_aead, ARRAY_SIZE(algs)); + crypto_engine_unregister_aeads(algs_aead, ARRAY_SIZE(algs_aead)); return ret; } @@ -1561,6 +1561,7 @@ static const struct of_device_id kmb_ocs_aes_of_match[] = { }, {} }; +MODULE_DEVICE_TABLE(of, kmb_ocs_aes_of_match); static void kmb_ocs_aes_remove(struct platform_device *pdev) { @@ -1602,6 +1603,8 @@ static int kmb_ocs_aes_probe(struct platform_device *pdev) if (IS_ERR(aes_dev->base_reg)) return PTR_ERR(aes_dev->base_reg); + init_completion(&aes_dev->irq_completion); + /* Get and request IRQ */ aes_dev->irq = platform_get_irq(pdev, 0); if (aes_dev->irq < 0) @@ -1619,8 +1622,6 @@ static int kmb_ocs_aes_probe(struct platform_device *pdev) list_add_tail(&aes_dev->list, &ocs_aes.dev_list); spin_unlock(&ocs_aes.lock); - init_completion(&aes_dev->irq_completion); - /* Initialize crypto engine */ aes_dev->engine = crypto_engine_alloc_init(dev, true); if (!aes_dev->engine) { diff --git a/drivers/crypto/intel/qat/qat_common/adf_aer.c b/drivers/crypto/intel/qat/qat_common/adf_aer.c index d58cd7fbf70771..afded3030e9a37 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_aer.c +++ b/drivers/crypto/intel/qat/qat_common/adf_aer.c @@ -189,6 +189,8 @@ static void adf_device_reset_worker(struct work_struct *work) queue_work(device_sriov_wq, &sriov_data.sriov_work); if (wait_for_completion_timeout(&sriov_data.compl, wait_jiffies)) adf_pf2vf_notify_restarted(accel_dev); + else + cancel_work_sync(&sriov_data.sriov_work); adf_dev_restarted_notify(accel_dev); clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg.c b/drivers/crypto/intel/qat/qat_common/adf_cfg.c index ea5d72d5090c03..b88febf53a19d9 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg.c +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg.c @@ -2,6 +2,7 @@ /* Copyright(c) 2014 - 2020 Intel Corporation */ #include #include +#include #include #include #include "adf_accel_devices.h" @@ -284,16 +285,13 @@ int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, return -ENOMEM; INIT_LIST_HEAD(&key_val->list); - strscpy(key_val->key, key, sizeof(key_val->key)); + strscpy(key_val->key, key); if (type == ADF_DEC) { snprintf(key_val->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES, "%ld", (*((long *)val))); } else if (type == ADF_STR) { - strscpy(key_val->val, (char *)val, sizeof(key_val->val)); - } else if (type == ADF_HEX) { - snprintf(key_val->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES, - "0x%lx", (unsigned long)val); + strscpy(key_val->val, (char *)val); } else { dev_err(&GET_DEV(accel_dev), "Unknown type given.\n"); kfree(key_val); @@ -350,7 +348,7 @@ int adf_cfg_section_add(struct adf_accel_dev *accel_dev, const char *name) if (!sec) return -ENOMEM; - strscpy(sec->name, name, sizeof(sec->name)); + strscpy(sec->name, name); INIT_LIST_HEAD(&sec->param_head); down_write(&cfg->lock); list_add_tail(&sec->list, &cfg->sec_list); diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h index d63f4dcccbb5d1..5922d018f5b989 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h @@ -28,7 +28,6 @@ enum adf_cfg_service_type { enum adf_cfg_val_type { ADF_DEC, - ADF_HEX, ADF_STR }; diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c index 1af6da8b263fbd..0cb6cb63e99508 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c @@ -60,7 +60,7 @@ static int adf_service_string_to_mask(struct adf_accel_dev *accel_dev, const cha if (len > ADF_CFG_MAX_VAL_LEN_IN_BYTES - 1) return -EINVAL; - strscpy(services, buf, ADF_CFG_MAX_VAL_LEN_IN_BYTES); + strscpy(services, buf); substr = services; while ((token = strsep(&substr, ADF_SERVICES_DELIMITER))) { diff --git a/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c b/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c index f9017e03ec0f29..32aeb795cc0361 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c +++ b/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c @@ -2,6 +2,7 @@ /* Copyright(c) 2024 Intel Corporation */ #include +#include #include #include "adf_mstate_mgr.h" @@ -158,7 +159,7 @@ static struct adf_mstate_sect_h *adf_mstate_sect_add_header(struct adf_mstate_mg return NULL; } - strscpy(sect->id, id, sizeof(sect->id)); + strscpy(sect->id, id); sect->size = 0; sect->sub_sects = 0; mgr->state += sizeof(*sect); diff --git a/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c b/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c index a8f853516a3f9f..fc5d88a2bb172c 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c +++ b/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c @@ -2,6 +2,7 @@ /* Copyright(c) 2014 - 2020 Intel Corporation */ #include #include +#include #include #include "adf_accel_devices.h" #include "adf_transport_internal.h" @@ -103,7 +104,7 @@ int adf_ring_debugfs_add(struct adf_etr_ring_data *ring, const char *name) if (!ring_debug) return -ENOMEM; - strscpy(ring_debug->ring_name, name, sizeof(ring_debug->ring_name)); + strscpy(ring_debug->ring_name, name); snprintf(entry_name, sizeof(entry_name), "ring_%02d", ring->ring_number); diff --git a/drivers/crypto/intel/qat/qat_common/qat_algs.c b/drivers/crypto/intel/qat/qat_common/qat_algs.c index 7f638a62e3adec..91663805d9e606 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_algs.c +++ b/drivers/crypto/intel/qat/qat_common/qat_algs.c @@ -405,6 +405,7 @@ static void qat_alg_xts_reverse_key(const u8 *key_forward, unsigned int keylen, memcpy(key_reverse + AES_BLOCK_SIZE, key - AES_BLOCK_SIZE, AES_BLOCK_SIZE); } + memzero_explicit(&aes_expanded, sizeof(aes_expanded)); } static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx, diff --git a/drivers/crypto/intel/qat/qat_common/qat_compression.c b/drivers/crypto/intel/qat/qat_common/qat_compression.c index 1424d7a9bcd361..8129ad0c32d84b 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_compression.c +++ b/drivers/crypto/intel/qat/qat_common/qat_compression.c @@ -2,6 +2,7 @@ /* Copyright(c) 2022 Intel Corporation */ #include #include +#include #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "adf_transport.h" @@ -144,7 +145,7 @@ static int qat_compression_create_instances(struct adf_accel_dev *accel_dev) int i; INIT_LIST_HEAD(&accel_dev->compression_list); - strscpy(key, ADF_NUM_DC, sizeof(key)); + strscpy(key, ADF_NUM_DC); ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); if (ret) return ret; diff --git a/drivers/crypto/rockchip/rk3288_crypto_ahash.c b/drivers/crypto/rockchip/rk3288_crypto_ahash.c index b9f5a8b42e6618..d3482619aa2f15 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_ahash.c +++ b/drivers/crypto/rockchip/rk3288_crypto_ahash.c @@ -324,7 +324,12 @@ static int rk_hash_run(struct crypto_engine *engine, void *breq) * efficiency, and make it response quickly when dma * complete. */ - readl_poll_timeout(rkc->reg + RK_CRYPTO_HASH_STS, v, v == 0, 10, 1000); + err = readl_poll_timeout(rkc->reg + RK_CRYPTO_HASH_STS, v, + v == 0, 10, 1000); + if (err) { + dev_err(rkc->dev, "HASH idle timeout\n"); + goto theend; + } for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++) { v = readl(rkc->reg + RK_CRYPTO_HASH_DOUT_0 + i * 4); diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index 965a03d5b27ae4..d865fd4a098cbc 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -2395,7 +2395,10 @@ static int sa_ul_probe(struct platform_device *pdev) return ret; } - sa_init_mem(dev_data); + ret = sa_init_mem(dev_data); + if (ret) + goto disable_pm; + ret = sa_dma_init(dev_data); if (ret) goto destroy_dma_pool; @@ -2430,6 +2433,7 @@ release_dma: destroy_dma_pool: dma_pool_destroy(dev_data->sc_pool); +disable_pm: pm_runtime_put_sync(dev); pm_runtime_disable(dev); diff --git a/drivers/cxl/core/features.c b/drivers/cxl/core/features.c index 738a89863ee896..ba6d2a5acb74ad 100644 --- a/drivers/cxl/core/features.c +++ b/drivers/cxl/core/features.c @@ -225,7 +225,7 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid, void *feat_out, size_t feat_out_size, u16 offset, u16 *return_code) { - size_t data_to_rd_size, size_out; + size_t data_to_rd_size; struct cxl_mbox_get_feat_in pi; struct cxl_mbox_cmd mbox_cmd; size_t data_rcvd_size = 0; @@ -237,9 +237,10 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid, if (!feat_out || !feat_out_size) return 0; - size_out = min(feat_out_size, cxl_mbox->payload_size); uuid_copy(&pi.uuid, feat_uuid); pi.selection = selection; + + guard(mutex)(&cxl_mbox->feat_mutex); do { data_to_rd_size = min(feat_out_size - data_rcvd_size, cxl_mbox->payload_size); @@ -250,7 +251,7 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid, .opcode = CXL_MBOX_OP_GET_FEATURE, .size_in = sizeof(pi), .payload_in = &pi, - .size_out = size_out, + .size_out = data_to_rd_size, .payload_out = feat_out + data_rcvd_size, .min_out = data_to_rd_size, }; @@ -314,6 +315,7 @@ int cxl_set_feature(struct cxl_mailbox *cxl_mbox, data_in_size = cxl_mbox->payload_size - hdr_size; } + guard(mutex)(&cxl_mbox->feat_mutex); do { int rc; @@ -471,6 +473,10 @@ static void *cxlctl_get_feature(struct cxl_features_state *cxlfs, if (!count) return ERR_PTR(-EINVAL); + if (out_size < offsetof(struct fwctl_rpc_cxl_out, payload) || + count > out_size - offsetof(struct fwctl_rpc_cxl_out, payload)) + return ERR_PTR(-EINVAL); + struct fwctl_rpc_cxl_out *rpc_out __free(kvfree) = kvzalloc(out_size, GFP_KERNEL); if (!rpc_out) @@ -516,6 +522,9 @@ static void *cxlctl_set_feature(struct cxl_features_state *cxlfs, flags = le32_to_cpu(feat_in->flags); out_size = *out_len; + if (out_size < offsetof(struct fwctl_rpc_cxl_out, payload)) + return ERR_PTR(-EINVAL); + struct fwctl_rpc_cxl_out *rpc_out __free(kvfree) = kvzalloc(out_size, GFP_KERNEL); if (!rpc_out) diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index 1fa1f78565e310..cc479f4322e73d 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -379,11 +379,7 @@ static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox_cmd, } } - /* Prepare to handle a full payload for variable sized output */ - if (out_size == CXL_VARIABLE_PAYLOAD) - mbox_cmd->size_out = cxl_mbox->payload_size; - else - mbox_cmd->size_out = out_size; + mbox_cmd->size_out = min_t(size_t, out_size, cxl_mbox->payload_size); if (mbox_cmd->size_out) { mbox_cmd->payload_out = kvzalloc(mbox_cmd->size_out, GFP_KERNEL); @@ -1454,6 +1450,11 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len, if (rc) break; + if (!le16_to_cpu(po->count)) { + dev_dbg(&cxlmd->dev, "Poison empty payload!\n"); + break; + } + for (int i = 0; i < le16_to_cpu(po->count); i++) trace_cxl_poison(cxlmd, cxlr, &po->record[i], po->flags, po->overflow_ts, @@ -1515,6 +1516,7 @@ int cxl_mailbox_init(struct cxl_mailbox *cxl_mbox, struct device *host) cxl_mbox->host = host; mutex_init(&cxl_mbox->mbox_mutex); + mutex_init(&cxl_mbox->feat_mutex); rcuwait_init(&cxl_mbox->mbox_wait); return 0; diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c index 33a3d2e7b13af9..45e7d2be17e09e 100644 --- a/drivers/cxl/core/memdev.c +++ b/drivers/cxl/core/memdev.c @@ -921,7 +921,7 @@ static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data, if (!size) return FW_UPLOAD_ERR_INVALID_SIZE; - mds->fw.oneshot = struct_size(transfer, data, size) < + mds->fw.oneshot = struct_size(transfer, data, size) <= cxl_mbox->payload_size; if (cxl_mem_get_fw_info(mds)) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 1215ee4f40351b..65f2d2f1eb0034 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -1749,8 +1749,8 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, parent_dport, uport_dev, dport_dev); if (IS_ERR(dport)) { - /* Port already exists, restart iteration */ - if (PTR_ERR(dport) == -EAGAIN) + /* Port or dport already exists, restart iteration */ + if (PTR_ERR(dport) == -EAGAIN || PTR_ERR(dport) == -EBUSY) return 0; return PTR_ERR(dport); } diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 578622240401d4..8b0005a57d03c1 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -1939,14 +1939,13 @@ static int find_pos_and_ways(struct cxl_port *port, struct range *range, break; } } - put_device(dev); - if (rc) dev_err(port->uport_dev, "failed to find %s:%s in target list of %s\n", dev_name(&port->dev), - dev_name(port->parent_dport->dport_dev), - dev_name(&cxlsd->cxld.dev)); + dev_name(port->parent_dport->dport_dev), dev_name(dev)); + + put_device(dev); return rc; } diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 7c6faee7f85ed5..d2f761f15a1156 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -691,12 +691,6 @@ static int cxl_pci_type3_init_mailbox(struct cxl_dev_state *cxlds) { int rc; - /* - * Fail the init if there's no mailbox. For a type3 this is out of spec. - */ - if (!cxlds->reg_map.device_map.mbox.valid) - return -ENODEV; - rc = cxl_mailbox_init(&cxlds->cxl_mbox, cxlds->dev); if (rc) return rc; @@ -829,10 +823,13 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) */ rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT, &cxlds->reg_map); - if (rc) + if (rc) { + if (rc == -EPROBE_DEFER) + return rc; dev_warn(&pdev->dev, "No component registers (%d)\n", rc); - else if (!cxlds->reg_map.component_map.ras.valid) + } else if (!cxlds->reg_map.component_map.ras.valid) { dev_dbg(&pdev->dev, "RAS registers not found\n"); + } rc = cxl_pci_type3_init_mailbox(cxlds); if (rc) diff --git a/drivers/dax/dax-private.h b/drivers/dax/dax-private.h index 81e4af49e39c10..607a53a91f58b3 100644 --- a/drivers/dax/dax-private.h +++ b/drivers/dax/dax-private.h @@ -69,7 +69,6 @@ struct dev_dax_range { * data while the device is activated in the driver. * @region: parent region * @dax_dev: core dax functionality - * @virt_addr: kva from memremap; used by fsdev_dax * @cached_size: size of daxdev cached by fsdev_dax * @align: alignment of this instance * @target_node: effective numa node if dev_dax memory range is onlined @@ -85,7 +84,6 @@ struct dev_dax_range { struct dev_dax { struct dax_region *region; struct dax_device *dax_dev; - void *virt_addr; u64 cached_size; unsigned int align; int target_node; diff --git a/drivers/dax/fsdev.c b/drivers/dax/fsdev.c index 188b2526bee455..d50891d6dc135b 100644 --- a/drivers/dax/fsdev.c +++ b/drivers/dax/fsdev.c @@ -51,9 +51,7 @@ static long __fsdev_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, struct dev_dax *dev_dax = dax_get_private(dax_dev); size_t size = nr_pages << PAGE_SHIFT; size_t offset = pgoff << PAGE_SHIFT; - void *virt_addr = dev_dax->virt_addr + offset; phys_addr_t phys; - unsigned long local_pfn; phys = dax_pgoff_to_phys(dev_dax, pgoff, size); if (phys == -1) { @@ -63,11 +61,10 @@ static long __fsdev_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, } if (kaddr) - *kaddr = virt_addr; + *kaddr = __va(phys); - local_pfn = PHYS_PFN(phys); if (pfn) - *pfn = local_pfn; + *pfn = PHYS_PFN(phys); /* * Use cached_size which was computed at probe time. The size cannot @@ -127,6 +124,23 @@ static void fsdev_clear_ops(void *data) dax_set_ops(dev_dax->dax_dev, NULL); } +static void fsdev_clear_pgmap_ops(void *data) +{ + struct dev_pagemap *pgmap = data; + + /* + * fsdev installs pgmap->ops and ->owner at probe. For a static device + * the pgmap is shared and long-lived (owned by the dax bus), so + * leaving fsdev's ops behind on unbind would let a later + * memory_failure -- after rebind to another driver, or after this + * module is unloaded -- dispatch through a stale or freed + * ->memory_failure handler. Clear them so the pgmap carries no fsdev + * state once we are unbound. + */ + pgmap->ops = NULL; + pgmap->owner = NULL; +} + /* * Page map operations for FS-DAX mode * Similar to fsdax_pagemap_ops in drivers/nvdimm/pmem.c @@ -135,11 +149,26 @@ static void fsdev_clear_ops(void *data) * The core mm code in free_zone_device_folio() handles the wake_up_var() * directly for this memory type. */ +static u64 fsdev_pfn_to_offset(struct dev_dax *dev_dax, unsigned long pfn) +{ + phys_addr_t phys = PFN_PHYS(pfn); + u64 offset = 0; + + for (int i = 0; i < dev_dax->nr_range; i++) { + struct range *range = &dev_dax->ranges[i].range; + + if (phys >= range->start && phys <= range->end) + return offset + (phys - range->start); + offset += range_len(range); + } + return -1ULL; +} + static int fsdev_pagemap_memory_failure(struct dev_pagemap *pgmap, unsigned long pfn, unsigned long nr_pages, int mf_flags) { struct dev_dax *dev_dax = pgmap->owner; - u64 offset = PFN_PHYS(pfn) - dev_dax->ranges[0].range.start; + u64 offset = fsdev_pfn_to_offset(dev_dax, pfn); u64 len = nr_pages << PAGE_SHIFT; return dax_holder_notify_failure(dev_dax->dax_dev, offset, @@ -204,46 +233,62 @@ static const struct file_operations fsdev_fops = { .release = fsdev_release, }; -static int fsdev_dax_probe(struct dev_dax *dev_dax) +/* + * Acquire the dev_pagemap for probe: the static (pre-populated) one if + * present, or a devm-allocated one for the dynamic case. Note that + * dev_dax->pgmap is not set here; fsdev_dax_probe() sets it only once + * probe succeeds, so a failed probe never leaves a dangling pointer + * to a devres-freed pgmap. + */ +static struct dev_pagemap *fsdev_acquire_pgmap(struct dev_dax *dev_dax) { - struct dax_device *dax_dev = dev_dax->dax_dev; struct device *dev = &dev_dax->dev; struct dev_pagemap *pgmap; - struct inode *inode; - u64 data_offset = 0; - struct cdev *cdev; - void *addr; - int rc, i; + size_t pgmap_size; if (static_dev_dax(dev_dax)) { if (dev_dax->nr_range > 1) { - dev_warn(dev, "static pgmap / multi-range device conflict\n"); - return -EINVAL; + dev_warn(dev, + "static pgmap / multi-range device conflict\n"); + return ERR_PTR(-EINVAL); } pgmap = dev_dax->pgmap; - } else { - size_t pgmap_size; + pgmap->vmemmap_shift = 0; + return pgmap; + } - if (dev_dax->pgmap) { - dev_warn(dev, "dynamic-dax with pre-populated page map\n"); - return -EINVAL; - } + if (dev_dax->pgmap) { + dev_warn(dev, "dynamic-dax with pre-populated page map\n"); + return ERR_PTR(-EINVAL); + } - pgmap_size = struct_size(pgmap, ranges, dev_dax->nr_range - 1); - pgmap = devm_kzalloc(dev, pgmap_size, GFP_KERNEL); - if (!pgmap) - return -ENOMEM; + pgmap_size = struct_size(pgmap, ranges, dev_dax->nr_range - 1); + pgmap = devm_kzalloc(dev, pgmap_size, GFP_KERNEL); + if (!pgmap) + return ERR_PTR(-ENOMEM); - pgmap->nr_range = dev_dax->nr_range; - dev_dax->pgmap = pgmap; + pgmap->nr_range = dev_dax->nr_range; + for (int i = 0; i < dev_dax->nr_range; i++) + pgmap->ranges[i] = dev_dax->ranges[i].range; - for (i = 0; i < dev_dax->nr_range; i++) { - struct range *range = &dev_dax->ranges[i].range; + return pgmap; +} - pgmap->ranges[i] = *range; - } - } +static int fsdev_dax_probe(struct dev_dax *dev_dax) +{ + struct dax_device *dax_dev = dev_dax->dax_dev; + struct device *dev = &dev_dax->dev; + struct dev_pagemap *pgmap; + struct inode *inode; + u64 data_offset = 0; + struct cdev *cdev; + void *addr; + int rc, i; + + pgmap = fsdev_acquire_pgmap(dev_dax); + if (IS_ERR(pgmap)) + return PTR_ERR(pgmap); for (i = 0; i < dev_dax->nr_range; i++) { struct range *range = &dev_dax->ranges[i].range; @@ -275,6 +320,11 @@ static int fsdev_dax_probe(struct dev_dax *dev_dax) if (IS_ERR(addr)) return PTR_ERR(addr); + /* Drop fsdev's pgmap->ops/owner on unbind so no stale ops survive. */ + rc = devm_add_action_or_reset(dev, fsdev_clear_pgmap_ops, pgmap); + if (rc) + return rc; + /* * Clear any stale compound folio state left over from a previous * driver (e.g., device_dax with vmemmap_shift). Also register this @@ -290,15 +340,18 @@ static int fsdev_dax_probe(struct dev_dax *dev_dax) /* Detect whether the data is at a non-zero offset into the memory */ if (pgmap->range.start != dev_dax->ranges[0].range.start) { u64 phys = dev_dax->ranges[0].range.start; - u64 pgmap_phys = dev_dax->pgmap[0].range.start; + u64 pgmap_phys = pgmap[0].range.start; - if (!WARN_ON(pgmap_phys > phys)) - data_offset = phys - pgmap_phys; + if (pgmap_phys > phys) { + dev_err(dev, "pgmap start %#llx exceeds data start %#llx\n", + pgmap_phys, phys); + return -EINVAL; + } + data_offset = phys - pgmap_phys; pr_debug("%s: offset detected phys=%llx pgmap_phys=%llx offset=%llx\n", __func__, phys, pgmap_phys, data_offset); } - dev_dax->virt_addr = addr + data_offset; inode = dax_inode(dax_dev); cdev = inode->i_cdev; @@ -323,7 +376,13 @@ static int fsdev_dax_probe(struct dev_dax *dev_dax) return rc; run_dax(dax_dev); - return devm_add_action_or_reset(dev, fsdev_kill, dev_dax); + rc = devm_add_action_or_reset(dev, fsdev_kill, dev_dax); + if (rc) + return rc; + + /* Probe can no longer fail; expose the pgmap via dev_dax */ + dev_dax->pgmap = pgmap; + return 0; } static struct dax_device_driver fsdev_dax_driver = { diff --git a/drivers/dax/super.c b/drivers/dax/super.c index 25cf99dd9360ba..45f84b0eb909a6 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -116,11 +116,47 @@ EXPORT_SYMBOL_GPL(fs_dax_get_by_bdev); #if IS_ENABLED(CONFIG_FS_DAX) +/** + * fs_put_dax() - release holder ownership of a dax_device + * @dax_dev: dax device to release (may be NULL) + * @holder: the holder pointer previously passed to fs_dax_get() or + * fs_dax_get_by_bdev(); must match exactly, as it is used + * in a cmpxchg to atomically release ownership + * + * Must only be called by the current holder. Clears holder_ops before + * holder_data to avoid a race where a concurrent fs_dax_get() could have + * its newly installed holder_ops overwritten. + */ void fs_put_dax(struct dax_device *dax_dev, void *holder) { - if (dax_dev && holder && - cmpxchg(&dax_dev->holder_data, holder, NULL) == holder) - dax_dev->holder_ops = NULL; + if (dax_dev && holder) { + void *prev; + + /* + * Clear holder_ops before releasing holder_data. A concurrent + * dax_holder_notify_failure() that sees NULL ops returns + * -EOPNOTSUPP cleanly. A concurrent fs_dax_get() that acquires + * holder_data after the cmpxchg below is guaranteed to observe + * holder_ops=NULL first (cmpxchg provides release ordering), so + * its subsequent store of new ops will not be overwritten. + */ + WRITE_ONCE(dax_dev->holder_ops, NULL); + prev = cmpxchg(&dax_dev->holder_data, holder, NULL); + + /* + * prev == holder: normal release. + * prev == NULL: already released by kill_dax() when the + * device was removed under a live holder; + * not a bug. + * prev != holder (non-NULL): fs_put_dax() called by something + * that is not the current holder; an API + * contract violation. A lock would be needed + * to guard against this, but we WARN_ON() + * instead since violating the contract is + * a bug. + */ + WARN_ON(prev && prev != holder); + } put_dax(dax_dev); } EXPORT_SYMBOL_GPL(fs_put_dax); @@ -303,6 +339,7 @@ EXPORT_SYMBOL_GPL(dax_recovery_write); int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, u64 len, int mf_flags) { + const struct dax_holder_operations *ops; int rc, id; id = dax_read_lock(); @@ -311,12 +348,19 @@ int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, goto out; } - if (!dax_dev->holder_ops) { + /* + * Read holder_ops once: a concurrent fs_put_dax() can clear it without + * synchronizing against readers. Without the single fetch the compiler + * could reload between the NULL check and the call and dereference a + * NULL ops. + */ + ops = READ_ONCE(dax_dev->holder_ops); + if (!ops) { rc = -EOPNOTSUPP; goto out; } - rc = dax_dev->holder_ops->notify_failure(dax_dev, off, len, mf_flags); + rc = ops->notify_failure(dax_dev, off, len, mf_flags); out: dax_read_unlock(id); return rc; diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 18ec188c991194..5fc025ab4a04aa 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -201,6 +202,35 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan) return 1; } +static void dw_edma_terminate_vdesc(struct virt_dma_desc *vd) +{ + list_del(&vd->node); + dma_cookie_complete(&vd->tx); + vchan_terminate_vdesc(vd); +} + +static void dw_edma_terminate_vdesc_list(struct list_head *head) +{ + struct virt_dma_desc *vd, *_vd; + + list_for_each_entry_safe(vd, _vd, head, node) + dw_edma_terminate_vdesc(vd); +} + +/* Must be called with vc.lock held. */ +static void dw_edma_terminate_all_descs(struct dw_edma_chan *chan) +{ + /* + * This order must not be reversed. Cookies are assigned when + * descriptors are submitted, so desc_issued contains older cookies + * than desc_submitted. Completing desc_submitted first could move + * chan->vc.chan.completed_cookie backwards when desc_issued is + * terminated afterwards. + */ + dw_edma_terminate_vdesc_list(&chan->vc.desc_issued); + dw_edma_terminate_vdesc_list(&chan->vc.desc_submitted); +} + static void dw_edma_device_caps(struct dma_chan *dchan, struct dma_slave_caps *caps) { @@ -272,6 +302,8 @@ static int dw_edma_device_pause(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); int err = 0; + guard(spinlock_irqsave)(&chan->vc.lock); + if (!chan->configured) err = -EPERM; else if (chan->status != EDMA_ST_BUSY) @@ -289,6 +321,8 @@ static int dw_edma_device_resume(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); int err = 0; + guard(spinlock_irqsave)(&chan->vc.lock); + if (!chan->configured) { err = -EPERM; } else if (chan->status != EDMA_ST_PAUSE) { @@ -309,25 +343,29 @@ static int dw_edma_device_terminate_all(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); int err = 0; + guard(spinlock_irqsave)(&chan->vc.lock); + if (!chan->configured) { - /* Do nothing */ + dw_edma_terminate_all_descs(chan); } else if (chan->status == EDMA_ST_PAUSE) { + dw_edma_terminate_all_descs(chan); chan->status = EDMA_ST_IDLE; - chan->configured = false; } else if (chan->status == EDMA_ST_IDLE) { - chan->configured = false; + dw_edma_terminate_all_descs(chan); } else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) { /* * The channel is in a false BUSY state, probably didn't * receive or lost an interrupt */ + dw_edma_terminate_all_descs(chan); chan->status = EDMA_ST_IDLE; - chan->configured = false; } else if (chan->request > EDMA_REQ_PAUSE) { err = -EPERM; } else { chan->request = EDMA_REQ_STOP; } + if (chan->status == EDMA_ST_IDLE) + chan->request = EDMA_REQ_NONE; return err; } @@ -337,11 +375,9 @@ static void dw_edma_device_issue_pending(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); unsigned long flags; - if (!chan->configured) - return; - spin_lock_irqsave(&chan->vc.lock, flags); - if (vchan_issue_pending(&chan->vc) && chan->request == EDMA_REQ_NONE && + if (chan->configured && vchan_issue_pending(&chan->vc) && + chan->request == EDMA_REQ_NONE && chan->status == EDMA_ST_IDLE) { chan->status = EDMA_ST_BUSY; dw_edma_start_transfer(chan); @@ -686,8 +722,7 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan) break; case EDMA_REQ_STOP: - list_del(&vd->node); - vchan_cookie_complete(vd); + dw_edma_terminate_all_descs(chan); chan->request = EDMA_REQ_NONE; chan->status = EDMA_ST_IDLE; break; @@ -706,14 +741,16 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) spin_lock_irqsave(&chan->vc.lock, flags); vd = vchan_next_desc(&chan->vc); - if (vd) { + if (vd && chan->request == EDMA_REQ_STOP) { + dw_edma_terminate_all_descs(chan); + } else if (vd) { dw_hdma_set_callback_result(vd, DMA_TRANS_ABORTED); list_del(&vd->node); vchan_cookie_complete(vd); } - spin_unlock_irqrestore(&chan->vc.lock, flags); chan->request = EDMA_REQ_NONE; chan->status = EDMA_ST_IDLE; + spin_unlock_irqrestore(&chan->vc.lock, flags); } static void dw_edma_emul_irq_ack(struct irq_data *d) @@ -865,21 +902,49 @@ static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) return 0; } -static void dw_edma_free_chan_resources(struct dma_chan *dchan) +static void dw_edma_wait_termination(struct dma_chan *dchan) { + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); unsigned long timeout = jiffies + msecs_to_jiffies(5000); - int ret; + bool stopping; + /* + * A STOP may be deferred to a later interrupt while the channel is still + * running. Wait until that handler completes the termination. + */ while (time_before(jiffies, timeout)) { - ret = dw_edma_device_terminate_all(dchan); - if (!ret) - break; + scoped_guard(spinlock_irqsave, &chan->vc.lock) + stopping = chan->request == EDMA_REQ_STOP; - if (time_after_eq(jiffies, timeout)) + if (!stopping) return; - cpu_relax(); + fsleep(1000); } + + dev_warn(chan->dw->chip->dev, + "timeout waiting for channel termination\n"); +} + +static void dw_edma_device_synchronize(struct dma_chan *dchan) +{ + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + + dw_edma_wait_termination(dchan); + vchan_synchronize(&chan->vc); +} + +static void dw_edma_free_chan_resources(struct dma_chan *dchan) +{ + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + + dw_edma_device_terminate_all(dchan); + dw_edma_device_synchronize(dchan); + + scoped_guard(spinlock_irqsave, &chan->vc.lock) + chan->configured = false; + + vchan_free_chan_resources(&chan->vc); } static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) @@ -976,6 +1041,7 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) dma->device_pause = dw_edma_device_pause; dma->device_resume = dw_edma_device_resume; dma->device_terminate_all = dw_edma_device_terminate_all; + dma->device_synchronize = dw_edma_device_synchronize; dma->device_issue_pending = dw_edma_device_issue_pending; dma->device_tx_status = dw_edma_device_tx_status; dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg; diff --git a/drivers/dma/hisi_dma.c b/drivers/dma/hisi_dma.c index 28bf818f9aa63c..c751a2e49e6dc3 100644 --- a/drivers/dma/hisi_dma.c +++ b/drivers/dma/hisi_dma.c @@ -983,7 +983,7 @@ static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id) hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, chan_num), GFP_KERNEL); if (!hdma_dev) - return -EINVAL; + return -ENOMEM; hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2]; hdma_dev->pdev = pdev; diff --git a/drivers/dma/mediatek/mtk-uart-apdma.c b/drivers/dma/mediatek/mtk-uart-apdma.c index c269d84d7bd2bd..f74e9a3285887f 100644 --- a/drivers/dma/mediatek/mtk-uart-apdma.c +++ b/drivers/dma/mediatek/mtk-uart-apdma.c @@ -531,7 +531,7 @@ static int mtk_uart_apdma_probe(struct platform_device *pdev) for (i = 0; i < mtkd->dma_requests; i++) { c = devm_kzalloc(mtkd->ddev.dev, sizeof(*c), GFP_KERNEL); if (!c) { - rc = -ENODEV; + rc = -ENOMEM; goto err_no_dma; } diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c index 1bb26af0405f3a..05a3b1f9e0c23d 100644 --- a/drivers/dma/qcom/bam_dma.c +++ b/drivers/dma/qcom/bam_dma.c @@ -1401,6 +1401,7 @@ static void bam_dma_remove(struct platform_device *pdev) struct bam_device *bdev = platform_get_drvdata(pdev); u32 i; + pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_force_suspend(&pdev->dev); of_dma_controller_free(pdev->dev.of_node); diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index 404235c1735384..74ad80d6c5a5fa 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -167,6 +167,8 @@ /* Delay loop counter to prevent hardware failure */ #define XILINX_DMA_LOOP_COUNT 1000000 +/* Delay between polls (avoid a delay of 0 to prevent CPU stalls) */ +#define XILINX_DMA_POLL_DELAY_US 10 /* AXI DMA Specific Registers/Offsets */ #define XILINX_DMA_REG_SRCDSTADDR 0x18 @@ -1324,7 +1326,8 @@ static int xilinx_dma_stop_transfer(struct xilinx_dma_chan *chan) /* Wait for the hardware to halt */ return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, - val & XILINX_DMA_DMASR_HALTED, 0, + val & XILINX_DMA_DMASR_HALTED, + XILINX_DMA_POLL_DELAY_US, XILINX_DMA_LOOP_COUNT); } @@ -1339,7 +1342,8 @@ static int xilinx_cdma_stop_transfer(struct xilinx_dma_chan *chan) u32 val; return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, - val & XILINX_DMA_DMASR_IDLE, 0, + val & XILINX_DMA_DMASR_IDLE, + XILINX_DMA_POLL_DELAY_US, XILINX_DMA_LOOP_COUNT); } @@ -1356,7 +1360,8 @@ static void xilinx_dma_start(struct xilinx_dma_chan *chan) /* Wait for the hardware to start */ err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, - !(val & XILINX_DMA_DMASR_HALTED), 0, + !(val & XILINX_DMA_DMASR_HALTED), + XILINX_DMA_POLL_DELAY_US, XILINX_DMA_LOOP_COUNT); if (err) { @@ -1794,7 +1799,8 @@ static int xilinx_dma_reset(struct xilinx_dma_chan *chan) /* Wait for the hardware to finish reset */ err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMACR, tmp, - !(tmp & XILINX_DMA_DMACR_RESET), 0, + !(tmp & XILINX_DMA_DMACR_RESET), + XILINX_DMA_POLL_DELAY_US, XILINX_DMA_LOOP_COUNT); if (err) { @@ -1893,8 +1899,10 @@ static irqreturn_t xilinx_mcdma_irq_handler(int irq, void *data) if (status & XILINX_MCDMA_IRQ_IOC_MASK) { spin_lock(&chan->lock); xilinx_dma_complete_descriptor(chan); - chan->idle = true; - chan->start_transfer(chan); + if (list_empty(&chan->active_list)) { + chan->idle = true; + chan->start_transfer(chan); + } spin_unlock(&chan->lock); } @@ -1950,8 +1958,10 @@ static irqreturn_t xilinx_dma_irq_handler(int irq, void *data) XILINX_DMA_DMASR_DLY_CNT_IRQ)) { spin_lock(&chan->lock); xilinx_dma_complete_descriptor(chan); - chan->idle = true; - chan->start_transfer(chan); + if (list_empty(&chan->active_list)) { + chan->idle = true; + chan->start_transfer(chan); + } spin_unlock(&chan->lock); } diff --git a/drivers/dma/xilinx/zynqmp_dma.c b/drivers/dma/xilinx/zynqmp_dma.c index f6a812e49ddca4..ca2dee0012c876 100644 --- a/drivers/dma/xilinx/zynqmp_dma.c +++ b/drivers/dma/xilinx/zynqmp_dma.c @@ -1170,9 +1170,9 @@ static void zynqmp_dma_remove(struct platform_device *pdev) dma_async_device_unregister(&zdev->common); zynqmp_dma_chan_remove(zdev->chan); - if (pm_runtime_active(zdev->dev)) - zynqmp_dma_runtime_suspend(zdev->dev); pm_runtime_disable(zdev->dev); + if (!pm_runtime_status_suspended(zdev->dev)) + zynqmp_dma_runtime_suspend(zdev->dev); } static const struct of_device_id zynqmp_dma_of_match[] = { diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c index 43d51d942eadd3..a320eeb829ad28 100644 --- a/drivers/dpll/dpll_core.c +++ b/drivers/dpll/dpll_core.c @@ -876,19 +876,25 @@ int dpll_pin_register(struct dpll_device *dpll, struct dpll_pin *pin, const struct dpll_pin_ops *ops, void *priv) { + const struct dpll_device_ops *dev_ops; int ret; if (WARN_ON(!ops) || WARN_ON(!ops->state_on_dpll_get) || WARN_ON(!ops->direction_get) || - WARN_ON(ops->measured_freq_get && - (!dpll_device_ops(dpll)->freq_monitor_get || - !dpll_device_ops(dpll)->freq_monitor_set)) || WARN_ON(ops->supported_ffo && !ops->ffo_get)) return -EINVAL; mutex_lock(&dpll_lock); + dev_ops = dpll_device_ops(dpll); + if (WARN_ON(ops->measured_freq_get && + (!dev_ops || !dev_ops->freq_monitor_get || + !dev_ops->freq_monitor_set))) { + ret = -EINVAL; + goto out_unlock; + } + /* * For pins identified via firmware (pin->fwnode), allow registration * even if the pin's (module, clock_id) differs from the target DPLL. @@ -1081,12 +1087,8 @@ EXPORT_SYMBOL_GPL(dpll_pin_ref_sync_pair_add); static struct dpll_device_registration * dpll_device_registration_first(struct dpll_device *dpll) { - struct dpll_device_registration *reg; - - reg = list_first_entry_or_null((struct list_head *)&dpll->registration_list, - struct dpll_device_registration, list); - WARN_ON(!reg); - return reg; + return list_first_entry_or_null((struct list_head *)&dpll->registration_list, + struct dpll_device_registration, list); } void *dpll_priv(struct dpll_device *dpll) @@ -1094,6 +1096,8 @@ void *dpll_priv(struct dpll_device *dpll) struct dpll_device_registration *reg; reg = dpll_device_registration_first(dpll); + if (!reg) + return NULL; return reg->priv; } @@ -1102,6 +1106,8 @@ const struct dpll_device_ops *dpll_device_ops(struct dpll_device *dpll) struct dpll_device_registration *reg; reg = dpll_device_registration_first(dpll); + if (!reg) + return NULL; return reg->ops; } diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c index afb31c0040382c..9e55745e33e4fa 100644 --- a/drivers/dpll/dpll_netlink.c +++ b/drivers/dpll/dpll_netlink.c @@ -66,6 +66,22 @@ static bool dpll_pin_available(struct dpll_pin *pin) return false; } +static bool dpll_device_registered(struct dpll_device *dpll) +{ + return dpll_device_ops(dpll); +} + +static struct dpll_pin_ref *dpll_pin_first_registered_ref(struct dpll_pin *pin) +{ + struct dpll_pin_ref *ref; + unsigned long i; + + xa_for_each(&pin->dpll_refs, i, ref) + if (dpll_device_registered(ref->dpll)) + return ref; + return NULL; +} + /** * dpll_msg_add_pin_handle - attach pin handle attribute to a given message * @msg: pointer to sk_buff message to attach a pin handle @@ -656,6 +672,8 @@ dpll_msg_add_pin_dplls(struct sk_buff *msg, struct dpll_pin *pin, int ret; xa_for_each(&pin->dpll_refs, index, ref) { + if (!dpll_device_registered(ref->dpll)) + continue; attr = nla_nest_start(msg, DPLL_A_PIN_PARENT_DEVICE); if (!attr) return -EMSGSIZE; @@ -700,9 +718,10 @@ dpll_cmd_pin_get_one(struct sk_buff *msg, struct dpll_pin *pin, int ret; ref = dpll_pin_own_dpll_ref_first(pin); + if (!ref || !dpll_device_registered(ref->dpll)) + ref = dpll_pin_first_registered_ref(pin); if (!ref) - ref = dpll_xa_ref_dpll_first(&pin->dpll_refs); - ASSERT_NOT_NULL(ref); + return -ENODEV; ret = dpll_msg_add_pin_handle(msg, pin); if (ret) @@ -1091,6 +1110,8 @@ dpll_pin_freq_set(struct dpll_pin *pin, struct nlattr *a, } xa_for_each(&pin->dpll_refs, i, ref) { + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if ((!ops->frequency_set || !ops->frequency_get) && ref->dpll->module == pin->module && @@ -1101,7 +1122,7 @@ dpll_pin_freq_set(struct dpll_pin *pin, struct nlattr *a, } } ref = dpll_pin_own_dpll_ref_first(pin); - if (!ref) { + if (!ref || !dpll_device_registered(ref->dpll)) { NL_SET_ERR_MSG(extack, "pin owner dpll not found"); return -ENODEV; } @@ -1117,6 +1138,8 @@ dpll_pin_freq_set(struct dpll_pin *pin, struct nlattr *a, return 0; xa_for_each(&pin->dpll_refs, i, ref) { + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if (!ops->frequency_set) continue; @@ -1138,6 +1161,8 @@ rollback: xa_for_each(&pin->dpll_refs, i, ref) { if (ref == failed) break; + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if (!ops->frequency_set) continue; @@ -1163,6 +1188,8 @@ dpll_pin_esync_set(struct dpll_pin *pin, struct nlattr *a, int ret; xa_for_each(&pin->dpll_refs, i, ref) { + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if ((!ops->esync_set || !ops->esync_get) && ref->dpll->module == pin->module && @@ -1173,7 +1200,7 @@ dpll_pin_esync_set(struct dpll_pin *pin, struct nlattr *a, } } ref = dpll_pin_own_dpll_ref_first(pin); - if (!ref) { + if (!ref || !dpll_device_registered(ref->dpll)) { NL_SET_ERR_MSG(extack, "pin owner dpll not found"); return -ENODEV; } @@ -1199,6 +1226,8 @@ dpll_pin_esync_set(struct dpll_pin *pin, struct nlattr *a, xa_for_each(&pin->dpll_refs, i, ref) { void *pin_dpll_priv; + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if (!ops->esync_set) continue; @@ -1224,6 +1253,8 @@ rollback: if (ref == failed) break; + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if (!ops->esync_set) continue; @@ -1262,7 +1293,7 @@ dpll_pin_ref_sync_state_set(struct dpll_pin *pin, return -EINVAL; } ref = dpll_pin_own_dpll_ref_first(pin); - if (!ref) { + if (!ref || !dpll_device_registered(ref->dpll)) { NL_SET_ERR_MSG(extack, "pin owner dpll not found"); return -ENODEV; } @@ -1283,6 +1314,8 @@ dpll_pin_ref_sync_state_set(struct dpll_pin *pin, if (state == old_state) return 0; xa_for_each(&pin->dpll_refs, i, ref) { + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if (!ops->ref_sync_set) continue; @@ -1307,6 +1340,8 @@ rollback: xa_for_each(&pin->dpll_refs, i, ref) { if (ref == failed) break; + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if (!ops->ref_sync_set) continue; @@ -1500,6 +1535,8 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin, struct nlattr *phase_adj_attr, } xa_for_each(&pin->dpll_refs, i, ref) { + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if ((!ops->phase_adjust_set || !ops->phase_adjust_get) && ref->dpll->module == pin->module && @@ -1509,7 +1546,7 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin, struct nlattr *phase_adj_attr, } } ref = dpll_pin_own_dpll_ref_first(pin); - if (!ref) { + if (!ref || !dpll_device_registered(ref->dpll)) { NL_SET_ERR_MSG(extack, "pin owner dpll not found"); return -ENODEV; } @@ -1526,6 +1563,8 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin, struct nlattr *phase_adj_attr, return 0; xa_for_each(&pin->dpll_refs, i, ref) { + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if (!ops->phase_adjust_set) continue; @@ -1550,6 +1589,8 @@ rollback: xa_for_each(&pin->dpll_refs, i, ref) { if (ref == failed) break; + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if (!ops->phase_adjust_set) continue; @@ -1581,7 +1622,7 @@ dpll_pin_parent_device_set(struct dpll_pin *pin, struct nlattr *parent_nest, return -EINVAL; } pdpll_idx = nla_get_u32(tb[DPLL_A_PIN_PARENT_ID]); - dpll = xa_load(&dpll_device_xa, pdpll_idx); + dpll = dpll_device_get_by_id(pdpll_idx); if (!dpll) { NL_SET_ERR_MSG(extack, "parent device not found"); return -EINVAL; @@ -1873,6 +1914,10 @@ int dpll_nl_pin_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) ret = dpll_cmd_pin_get_one(skb, pin, cb->extack); if (ret) { genlmsg_cancel(skb, hdr); + if (ret == -ENODEV) { + ret = 0; + continue; + } break; } genlmsg_end(skb, hdr); diff --git a/drivers/firewire/.kunitconfig b/drivers/firewire/.kunitconfig index 21b7e9eef63d1e..7406acb0047821 100644 --- a/drivers/firewire/.kunitconfig +++ b/drivers/firewire/.kunitconfig @@ -6,3 +6,4 @@ CONFIG_FIREWIRE_KUNIT_DEVICE_ATTRIBUTE_TEST=y CONFIG_FIREWIRE_KUNIT_PACKET_SERDES_TEST=y CONFIG_FIREWIRE_KUNIT_SELF_ID_SEQUENCE_HELPER_TEST=y CONFIG_FIREWIRE_KUNIT_OHCI_SERDES_TEST=y +CONFIG_FIREWIRE_KUNIT_NODE_TREE_TEST=y diff --git a/drivers/firewire/Kconfig b/drivers/firewire/Kconfig index a5f5e250223a18..b5abe00accc9dd 100644 --- a/drivers/firewire/Kconfig +++ b/drivers/firewire/Kconfig @@ -81,6 +81,21 @@ config FIREWIRE_KUNIT_SELF_ID_SEQUENCE_HELPER_TEST For more information on KUnit and unit tests in general, refer to the KUnit documentation in Documentation/dev-tools/kunit/. +config FIREWIRE_KUNIT_NODE_TREE_TEST + tristate "KUnit tests for node tree" if !KUNIT_ALL_TESTS + depends on FIREWIRE && KUNIT + default KUNIT_ALL_TESTS + help + This builds the KUnit tests for node tree. + + KUnit tests run during boot and output the results to the debug + log in TAP format (https://testanything.org/). Only useful for + kernel devs running KUnit test harness and are not for inclusion + into a production build. + + For more information on KUnit and unit tests in general, refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + config FIREWIRE_OHCI tristate "OHCI-1394 controllers" depends on PCI && FIREWIRE diff --git a/drivers/firewire/core-topology.c b/drivers/firewire/core-topology.c index df2ac0dab10605..ee6b54f89859ef 100644 --- a/drivers/firewire/core-topology.c +++ b/drivers/firewire/core-topology.c @@ -88,6 +88,17 @@ static inline struct fw_node *fw_node(struct list_head *l) return list_entry(l, struct fw_node, link); } +typedef void (*fw_node_callback_t)(struct fw_card *card, struct fw_node *node, + struct fw_node *parent); + +static void for_each_fw_node(struct fw_card *card, struct fw_node *root, + fw_node_callback_t callback); + +static void free_fw_node(struct fw_card *card, struct fw_node *node, struct fw_node *parent) +{ + kfree(node); +} + /* * This function builds the tree representation of the topology given * by the self IDs from the latest bus reset. During the construction @@ -119,8 +130,8 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self while (enumerator.quadlet_count > 0) { unsigned int child_port_count = 0; + unsigned int parent_port_count = 0; unsigned int total_port_count = 0; - unsigned int parent_count = 0; unsigned int quadlet_count; const u32 *self_id_sequence; unsigned int port_capacity; @@ -134,7 +145,7 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self if (PTR_ERR(self_id_sequence) != -ENODATA) { fw_err(card, "inconsistent extended self IDs: %ld\n", PTR_ERR(self_id_sequence)); - return NULL; + goto error; } break; } @@ -148,26 +159,38 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self switch (port_status) { case PHY_PACKET_SELF_ID_PORT_STATUS_CHILD: ++child_port_count; - fallthrough; + break; case PHY_PACKET_SELF_ID_PORT_STATUS_PARENT: + ++parent_port_count; + break; case PHY_PACKET_SELF_ID_PORT_STATUS_NCONN: ++total_port_count; - fallthrough; + break; case PHY_PACKET_SELF_ID_PORT_STATUS_NONE: default: break; } } + total_port_count += child_port_count + parent_port_count; + + // Check that the node reports exactly one parent port, except for the root, which + // of course should have no parents. + if ((enumerator.quadlet_count == 0 && parent_port_count != 0) || + (enumerator.quadlet_count > 0 && parent_port_count != 1)) { + fw_err(card, "parent port inconsistency for node %d: parent_count=%d\n", + phy_id, parent_port_count); + goto error; + } if (phy_id != phy_packet_self_id_get_phy_id(self_id_sequence[0])) { fw_err(card, "PHY ID mismatch in self ID: %d != %d\n", phy_id, phy_packet_self_id_get_phy_id(self_id_sequence[0])); - return NULL; + goto error; } if (child_port_count > stack_depth) { fw_err(card, "topology stack underflow\n"); - return NULL; + goto error; } /* @@ -185,7 +208,7 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self node = fw_node_create(self_id_sequence[0], total_port_count, card->color); if (node == NULL) { fw_err(card, "out of memory while building topology\n"); - return NULL; + goto error; } if (phy_id == (card->node_id & 0x3f)) @@ -203,7 +226,6 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self // we temporarily abuse node->color for remembering the entry in // the node->ports array where the parent node should be. Later, // when we handle the parent node, we fix up the reference. - ++parent_count; node->color = port_index; break; @@ -221,21 +243,12 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self } } - // Check that the node reports exactly one parent port, except for the root, which - // of course should have no parents. - if ((enumerator.quadlet_count == 0 && parent_count != 0) || - (enumerator.quadlet_count > 0 && parent_count != 1)) { - fw_err(card, "parent port inconsistency for node %d: " - "parent_count=%d\n", phy_id, parent_count); - return NULL; - } - /* Pop the child nodes off the stack and push the new node. */ __list_del(h->prev, &stack); list_add_tail(&node->link, &stack); stack_depth += 1 - child_port_count; - if (node->phy_speed == SCODE_BETA && parent_count + child_port_count > 1) + if (node->phy_speed == SCODE_BETA && parent_port_count + child_port_count > 1) beta_repeaters_present = true; // If PHYs report different gap counts, set an invalid count which will force a gap @@ -254,12 +267,13 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self card->beta_repeaters_present = beta_repeaters_present; return local_node; +error: + ++card->color; + list_for_each_entry_safe(node, child, &stack, link) + for_each_fw_node(card, node, free_fw_node); + return NULL; } -typedef void (*fw_node_callback_t)(struct fw_card * card, - struct fw_node * node, - struct fw_node * parent); - static void for_each_fw_node(struct fw_card *card, struct fw_node *root, fw_node_callback_t callback) { @@ -507,3 +521,7 @@ void fw_core_handle_bus_reset(struct fw_card *card, int node_id, int generation, } } EXPORT_SYMBOL(fw_core_handle_bus_reset); + +#ifdef CONFIG_FIREWIRE_KUNIT_NODE_TREE_TEST +#include "node-tree-test.c" +#endif diff --git a/drivers/firewire/node-tree-test.c b/drivers/firewire/node-tree-test.c new file mode 100644 index 00000000000000..5e6bb58537e946 --- /dev/null +++ b/drivers/firewire/node-tree-test.c @@ -0,0 +1,607 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// node-tree-test.c - An application of Kunit to test node tree. +// +// Copyright (c) 2026 Takashi Sakamoto +// +// This file can not be built independently since it is intentionally included in core-topology.c. + +#include +#include +#include + +struct private_data { + struct fw_card *card; + unsigned int release_count; +}; + +static int node_tree_test_init(struct kunit *test) +{ + struct private_data *data; + + data = kunit_kzalloc(test, sizeof(*data), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, data); + + data->card = kunit_kzalloc(test, sizeof(struct fw_card), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, data->card); + + data->card->device = kunit_device_register(test, "dummy-device"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, data->card->device); + + test->priv = data; + + return 0; +} + +static void node_tree_test_exit(struct kunit *test) +{ + struct private_data *data = test->priv; + + kunit_device_unregister(test, data->card->device); + kunit_kfree(test, data->card); + kunit_kfree(test, data); +} + +static void release_fw_node(struct fw_card *card, struct fw_node *node, struct fw_node *parent) +{ + struct private_data *data = kunit_get_current_test()->priv; + + fw_node_put(node); + ++data->release_count; +} + +static void node_tree_test_two_nodes(struct kunit *test) +{ + // root + // ++============++ + // || phy 1 || + // || P0 P1 P2 || + // ++===|==|==|==++ + // | + // +-----+ + // | + // ++===|==x==x==++ + // || P0 P1 P2 || + // || phy 0 || + // ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000080, + 0x8100005e, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x01; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + struct fw_node *parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 1); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 2); +} + +static void node_tree_test_two_nodes_1394a(struct kunit *test) +{ + // root + // ++===============++ + // || phy 0 || + // || P0 P1 P2 P3 || + // ++===|==|==|==|==++ + // | + // +--+ + // | + // ++===|==|==|==|==|==++ + // || P0 P1 P2 P3 P4 || + // || phy 1 || + // ++==================++ + // + // NOTE: Just for Self-ID Packets Zero and One. + static const u32 self_id_sequence[] = { + 0x80000065, 0x80814000, + 0x8100005d, 0x81810000, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x01; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 4); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + KUNIT_EXPECT_NULL(test, node->ports[3]); + + struct fw_node *parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 5); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[1], parent); + KUNIT_EXPECT_NULL(test, node->ports[2]); + KUNIT_EXPECT_NULL(test, node->ports[3]); + KUNIT_EXPECT_NULL(test, node->ports[4]); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 2); +} + +static void node_tree_test_three_nodes_case0(struct kunit *test) +{ + // root + // ++============++ + // || phy 2 || + // || P0 P1 P2 || + // ++===|==|==|==++ + // | | + // +--+ +-----------------+ + // | | + // ++===|==|==x==++ ++===|==|==|==++ + // || P0 P1 P2 || || P0 P1 P2 || + // || phy 0 || || phy 1 || + // ++============++ ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000060, + 0x81000058, + 0x820000dc, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x02; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x02); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NOT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + struct fw_node *parent = node; + node = parent->ports[0]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[1], parent); + + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[2], parent); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 3); +} + +static void node_tree_test_three_nodes_case1(struct kunit *test) +{ + // root + // ++============++ + // || phy 2 || + // || P0 P1 P2 || + // ++===|==|==x==++ + // | + // | +-----------+ + // | | | + // ++===|==|==|==++ ++===|==x==x==++ + // || P0 P1 P2 || || P0 P1 P2 || + // || phy 1 || || phy 0 || + // ++============++ ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000080, + 0x8100006c, + 0x82000070, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x02; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x02); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[1]); + + struct fw_node *parent = node; + node = parent->ports[1]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[1], parent); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 1); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 3); +} + +static void node_tree_test_four_nodes_case0(struct kunit *test) +{ + // root + // ++============++ + // || phy 3 || + // || P0 P1 P2 || + // ++===|==|==|==++ + // | + // | +-----------+ +--------------+ + // | | | | | + // ++===|==|==|==++ ++===|==|==x==++ ++===|==x==x==++ + // || P0 P1 P2 || || P0 P1 P2 || || P0 P1 P2 || + // || phy 2 || || phy 1 || || phy 0 || + // ++============++ ++============++ ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000080, + 0x810000b0, + 0x8200006c, + 0x83000074, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x03); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NULL(test, node->ports[2]); + + struct fw_node *parent = node; + node = parent->ports[1]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x02); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[1], parent); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + KUNIT_EXPECT_NOT_NULL(test, node->ports[1]); + + parent = node; + node = parent->ports[1]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 1); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 4); +} + +static void node_tree_test_four_nodes_case1(struct kunit *test) +{ + // root + // ++============++ + // || phy 3 || + // || P0 P1 P2 || + // ++===|==|==x==++ + // | + // | +--------------------------------+ + // | | +-----------+ | + // ++===|==|==|==++ ++===|==x==x==++ ++===|==|==|==++ + // || P0 P1 P2 || || P0 P1 P2 || || P0 P1 P2 || + // || phy 2 || || phy 1 || || phy 0 || + // ++============++ ++============++ ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000094, + 0x81000080, + 0x820000bc, + 0x830000d0, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x03); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_NOT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + + struct fw_node *parent = node; + node = parent->ports[0]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x02); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + KUNIT_EXPECT_NOT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 1); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + + node = parent->ports[1]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NULL(test, node->ports[2]); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 4); +} + +static void node_tree_test_four_nodes_case2(struct kunit *test) +{ + // root + // ++============++ + // || phy 3 || + // || P0 P1 P2 || + // ++===|==|==|==++ + // | | + // | +-----------------------------+ + // | +--------------+ | + // ++===|==|==x==++ ++===|==|==|==++ ++===|==x==x==++ + // || P0 P1 P2 || || P0 P1 P2 || || P0 P1 P2 || + // || phy 1 || || phy 0 || || phy 2 || + // ++============++ ++============++ ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000094, + 0x810000b0, + 0x82000080, + 0x830000dc, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x03); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NOT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + struct fw_node *parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x02); + KUNIT_EXPECT_EQ(test, node->port_count, 1); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + + node = parent->ports[0]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + KUNIT_EXPECT_NOT_NULL(test, node->ports[1]); + + parent = node; + node = parent->ports[1]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NULL(test, node->ports[2]); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 4); +} + +static void node_tree_test_four_nodes_case3(struct kunit *test) +{ + // root + // ++============++ + // || phy 3 || + // || P0 P1 P2 || + // ++===|==|==|==++ + // | | +--------------------------------+ + // | +--------------------+ | + // | | | + // ++===|==|==x==++ ++===|==|==|==++ ++===|==|==x==++ + // || P0 P1 P2 || || P0 P1 P2 || || P0 P1 P2 || + // || phy 0 || || phy 1 || || phy 2 || + // ++============++ ++============++ ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000090, + 0x81000058, + 0x82000060, + 0x830000fc, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x03); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NOT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + struct fw_node *parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x02); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[1], parent); + + node = parent->ports[1]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[2], parent); + + node = parent->ports[0]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + KUNIT_EXPECT_NULL(test, node->ports[1]); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 4); +} + +static void node_tree_test_invalid_extended_self_id_sequence(struct kunit *test) +{ + // Use the same node tree as node_tree_test_four_nodes_case1, except for the invalid + // content of self ID packet for the phy 3. + static const u32 self_id_sequence[] = { + 0x80000094, + 0x81000080, + 0x820000bc, + 0x830000d1, // Invalid. + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NULL(test, card->local_node); +} + +static void node_tree_test_invalid_phy_id(struct kunit *test) +{ + // Use the same node tree as node_tree_test_four_nodes_case1, except for the invalid + // phy ID for phy 3. + static const u32 self_id_sequence[] = { + 0x80000094, + 0x81000080, + 0x820000bc, + 0x8f0000d0, // Invalid. + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NULL(test, card->local_node); +} + +static void node_tree_test_invalid_child_port_count(struct kunit *test) +{ + // Use the same node tree as node_tree_test_four_nodes_case1, except for the invalid + // count of child ports for phy 3. + static const u32 self_id_sequence[] = { + 0x80000094, + 0x81000080, + 0x820000bc, + 0x830000fc, // Invalid. + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NULL(test, card->local_node); +} + +static void node_tree_test_invalid_parent_port_count(struct kunit *test) +{ + // Use the same node tree as node_tree_test_four_nodes_case1, except for the invalid + // count of parent ports for phy 3. + static const u32 self_id_sequence[] = { + 0x80000094, + 0x81000080, + 0x820000bc, + 0x830000e8, // Invalid. + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NULL(test, card->local_node); +} + +static struct kunit_case node_tree_test_cases[] = { + KUNIT_CASE(node_tree_test_two_nodes), + KUNIT_CASE(node_tree_test_two_nodes_1394a), + KUNIT_CASE(node_tree_test_three_nodes_case0), + KUNIT_CASE(node_tree_test_three_nodes_case1), + KUNIT_CASE(node_tree_test_four_nodes_case0), + KUNIT_CASE(node_tree_test_four_nodes_case1), + KUNIT_CASE(node_tree_test_four_nodes_case2), + KUNIT_CASE(node_tree_test_four_nodes_case3), + KUNIT_CASE(node_tree_test_invalid_extended_self_id_sequence), + KUNIT_CASE(node_tree_test_invalid_phy_id), + KUNIT_CASE(node_tree_test_invalid_child_port_count), + KUNIT_CASE(node_tree_test_invalid_parent_port_count), + {} +}; + +static struct kunit_suite node_tree_test_suite = { + .name = "firewire-node-tree", + .init = node_tree_test_init, + .exit = node_tree_test_exit, + .test_cases = node_tree_test_cases, +}; +kunit_test_suite(node_tree_test_suite); diff --git a/drivers/firmware/arm_scmi/bus.c b/drivers/firmware/arm_scmi/bus.c index 793be9eabaedda..e060edbe7e832e 100644 --- a/drivers/firmware/arm_scmi/bus.c +++ b/drivers/firmware/arm_scmi/bus.c @@ -7,7 +7,6 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include #include #include #include @@ -33,8 +32,8 @@ struct scmi_requested_dev { struct list_head node; }; -/* Track globally the creation of SCMI SystemPower related devices */ -static atomic_t scmi_syspower_registered = ATOMIC_INIT(0); +/* Track globally the SCMI SystemPower protocol device. */ +static struct scmi_device *scmi_syspower_registered; /** * scmi_protocol_device_request - Helper to request a device @@ -136,17 +135,6 @@ out: return ret; } -static int scmi_protocol_table_register(const struct scmi_device_id *id_table) -{ - int ret = 0; - const struct scmi_device_id *entry; - - for (entry = id_table; entry->name && ret == 0; entry++) - ret = scmi_protocol_device_request(entry); - - return ret; -} - /** * scmi_protocol_device_unrequest - Helper to unrequest a device * @@ -159,6 +147,7 @@ static int scmi_protocol_table_register(const struct scmi_device_id *id_table) */ static void scmi_protocol_device_unrequest(const struct scmi_device_id *id_table) { + struct scmi_requested_dev *rdev, *victim = NULL; struct list_head *phead; pr_debug("Unrequesting SCMI device (%s) for protocol %x\n", @@ -167,29 +156,48 @@ static void scmi_protocol_device_unrequest(const struct scmi_device_id *id_table mutex_lock(&scmi_requested_devices_mtx); phead = idr_find(&scmi_requested_devices, id_table->protocol_id); if (phead) { - struct scmi_requested_dev *victim, *tmp; - - list_for_each_entry_safe(victim, tmp, phead, node) { - if (!strcmp(victim->id_table->name, id_table->name)) { - list_del(&victim->node); - - mutex_unlock(&scmi_requested_devices_mtx); - blocking_notifier_call_chain(&scmi_requested_devices_nh, - SCMI_BUS_NOTIFY_DEVICE_UNREQUEST, - (void *)victim->id_table); - kfree(victim); - mutex_lock(&scmi_requested_devices_mtx); + list_for_each_entry(rdev, phead, node) { + if (!strcmp(rdev->id_table->name, id_table->name)) { + victim = rdev; + list_del(&rdev->node); break; } } - if (list_empty(phead)) { + if (victim && list_empty(phead)) { idr_remove(&scmi_requested_devices, id_table->protocol_id); kfree(phead); } } mutex_unlock(&scmi_requested_devices_mtx); + + if (victim) { + blocking_notifier_call_chain(&scmi_requested_devices_nh, + SCMI_BUS_NOTIFY_DEVICE_UNREQUEST, + (void *)victim->id_table); + kfree(victim); + } +} + +static int scmi_protocol_table_register(const struct scmi_device_id *id_table) +{ + const struct scmi_device_id *entry; + int ret; + + for (entry = id_table; entry->name; entry++) { + ret = scmi_protocol_device_request(entry); + if (ret) + goto err_unrequest; + } + + return 0; + +err_unrequest: + while (entry != id_table) + scmi_protocol_device_unrequest(--entry); + + return ret; } static void @@ -201,21 +209,33 @@ scmi_protocol_table_unregister(const struct scmi_device_id *id_table) scmi_protocol_device_unrequest(entry); } -static int scmi_dev_match_by_id_table(struct scmi_device *scmi_dev, - const struct scmi_device_id *id_table) +static bool scmi_device_is_transport(const struct scmi_device *scmi_dev) +{ + return !strncmp(scmi_dev->name, SCMI_TRANSPORT_DEVNAME_PREFIX, + strlen(SCMI_TRANSPORT_DEVNAME_PREFIX)); +} + +static int __scmi_dev_match_by_id_table(struct scmi_device *scmi_dev, + const struct scmi_device_id *id_table, + bool skip_transport) { if (!id_table || !id_table->name) return 0; - /* Always skip transport devices from matching */ for (; id_table->protocol_id && id_table->name; id_table++) if (id_table->protocol_id == scmi_dev->protocol_id && - strncmp(scmi_dev->name, "__scmi_transport_device", 23) && + !(skip_transport && scmi_device_is_transport(scmi_dev)) && !strcmp(id_table->name, scmi_dev->name)) return 1; return 0; } +static int scmi_dev_match_by_id_table(struct scmi_device *scmi_dev, + const struct scmi_device_id *id_table) +{ + return __scmi_dev_match_by_id_table(scmi_dev, id_table, true); +} + static int scmi_dev_match_id(struct scmi_device *scmi_dev, const struct scmi_driver *scmi_drv) { @@ -235,11 +255,12 @@ static int scmi_match_by_id_table(struct device *dev, const void *data) struct scmi_device *scmi_dev = to_scmi_dev(dev); const struct scmi_device_id *id_table = data; - return scmi_dev_match_by_id_table(scmi_dev, id_table); + return __scmi_dev_match_by_id_table(scmi_dev, id_table, false); } -static struct scmi_device *scmi_child_dev_find(struct device *parent, - int prot_id, const char *name) +/* Returns a device_find_child() reference which must be dropped by caller. */ +static struct scmi_device * +scmi_child_dev_find_get(struct device *parent, int prot_id, const char *name) { struct scmi_device_id id_table[2] = { 0 }; struct device *dev; @@ -251,9 +272,6 @@ static struct scmi_device *scmi_child_dev_find(struct device *parent, if (!dev) return NULL; - /* Drop the refcnt bumped implicitly by device_find_child */ - put_device(dev); - return to_scmi_dev(dev); } @@ -377,10 +395,14 @@ int scmi_driver_register(struct scmi_driver *driver, struct module *owner, driver->driver.mod_name = mod_name; retval = driver_register(&driver->driver); - if (!retval) - pr_debug("Registered new scmi driver %s\n", driver->name); + if (retval) { + scmi_protocol_table_unregister(driver->id_table); + return retval; + } - return retval; + pr_debug("Registered new scmi driver %s\n", driver->name); + + return 0; } EXPORT_SYMBOL_GPL(scmi_driver_register); @@ -391,10 +413,23 @@ void scmi_driver_unregister(struct scmi_driver *driver) } EXPORT_SYMBOL_GPL(scmi_driver_unregister); +static void scmi_device_release_resources(struct scmi_device *scmi_dev) +{ + if (scmi_dev->protocol_id == SCMI_PROTOCOL_SYSTEM) + cmpxchg(&scmi_syspower_registered, scmi_dev, NULL); + + if (scmi_dev->id) { + ida_free(&scmi_bus_id, scmi_dev->id); + scmi_dev->id = 0; + } +} + static void scmi_device_release(struct device *dev) { struct scmi_device *scmi_dev = to_scmi_dev(dev); + scmi_device_release_resources(scmi_dev); + of_node_put(dev->of_node); kfree_const(scmi_dev->name); kfree(scmi_dev); } @@ -406,11 +441,9 @@ static void __scmi_device_destroy(struct scmi_device *scmi_dev) dev_name(&scmi_dev->dev), scmi_dev->protocol_id, scmi_dev->name); - if (scmi_dev->protocol_id == SCMI_PROTOCOL_SYSTEM) - atomic_set(&scmi_syspower_registered, 0); - - ida_free(&scmi_bus_id, scmi_dev->id); - device_unregister(&scmi_dev->dev); + device_del(&scmi_dev->dev); + scmi_device_release_resources(scmi_dev); + put_device(&scmi_dev->dev); } static struct scmi_device * @@ -419,6 +452,7 @@ __scmi_device_create(struct device_node *np, struct device *parent, { int id, retval; struct scmi_device *scmi_dev; + bool syspower = (protocol == SCMI_PROTOCOL_SYSTEM); /* * If the same protocol/name device already exist under the same parent @@ -427,45 +461,41 @@ __scmi_device_create(struct device_node *np, struct device *parent, * each DT defined protocol at probe time, and the concurrent * registration of SCMI drivers. */ - scmi_dev = scmi_child_dev_find(parent, protocol, name); - if (scmi_dev) + scmi_dev = scmi_child_dev_find_get(parent, protocol, name); + if (scmi_dev) { + put_device(&scmi_dev->dev); return scmi_dev; - - /* - * Ignore any possible subsequent failures while creating the device - * since we are doomed anyway at that point; not using a mutex which - * spans across this whole function to keep things simple and to avoid - * to serialize all the __scmi_device_create calls across possibly - * different SCMI server instances (parent) - */ - if (protocol == SCMI_PROTOCOL_SYSTEM && - atomic_cmpxchg(&scmi_syspower_registered, 0, 1)) { - dev_warn(parent, - "SCMI SystemPower protocol device must be unique !\n"); - return NULL; } scmi_dev = kzalloc_obj(*scmi_dev); if (!scmi_dev) return NULL; - scmi_dev->name = kstrdup_const(name ?: "unknown", GFP_KERNEL); - if (!scmi_dev->name) { + scmi_dev->protocol_id = protocol; + + /* + * Reserve the singleton SystemPower protocol device using the device + * pointer itself, so delayed release of an older device cannot clear + * a reservation owned by a newer device. + */ + if (syspower && cmpxchg(&scmi_syspower_registered, NULL, scmi_dev)) { + dev_warn(parent, + "SCMI SystemPower protocol device must be unique !\n"); kfree(scmi_dev); return NULL; } + scmi_dev->name = kstrdup_const(name ?: "unknown", GFP_KERNEL); + if (!scmi_dev->name) + goto free_dev; + id = ida_alloc_min(&scmi_bus_id, 1, GFP_KERNEL); - if (id < 0) { - kfree_const(scmi_dev->name); - kfree(scmi_dev); - return NULL; - } + if (id < 0) + goto free_name; scmi_dev->id = id; - scmi_dev->protocol_id = protocol; scmi_dev->dev.parent = parent; - device_set_node(&scmi_dev->dev, of_fwnode_handle(np)); + device_set_node(&scmi_dev->dev, of_fwnode_handle(of_node_get(np))); scmi_dev->dev.bus = &scmi_bus_type; scmi_dev->dev.release = scmi_device_release; dev_set_name(&scmi_dev->dev, "scmi_dev.%d", id); @@ -479,8 +509,14 @@ __scmi_device_create(struct device_node *np, struct device *parent, return scmi_dev; put_dev: + scmi_device_release_resources(scmi_dev); put_device(&scmi_dev->dev); - ida_free(&scmi_bus_id, id); + return NULL; +free_name: + kfree_const(scmi_dev->name); +free_dev: + scmi_device_release_resources(scmi_dev); + kfree(scmi_dev); return NULL; } @@ -561,9 +597,11 @@ void scmi_device_destroy(struct device *parent, int protocol, const char *name) { struct scmi_device *scmi_dev; - scmi_dev = scmi_child_dev_find(parent, protocol, name); - if (scmi_dev) + scmi_dev = scmi_child_dev_find_get(parent, protocol, name); + if (scmi_dev) { __scmi_device_destroy(scmi_dev); + put_device(&scmi_dev->dev); + } } EXPORT_SYMBOL_GPL(scmi_device_destroy); diff --git a/drivers/firmware/arm_scmi/common.h b/drivers/firmware/arm_scmi/common.h index b9723c105fc1b5..fe8c22cfb9f7d1 100644 --- a/drivers/firmware/arm_scmi/common.h +++ b/drivers/firmware/arm_scmi/common.h @@ -34,6 +34,8 @@ #define SCMI_SHMEM_MAX_PAYLOAD_SIZE 104 +#define SCMI_TRANSPORT_DEVNAME_PREFIX "__scmi_transport_device" + enum scmi_error_codes { SCMI_SUCCESS = 0, /* Success */ SCMI_ERR_SUPPORT = -1, /* Not supported */ diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c index 3e0d975ec94c44..ef29fd223287dd 100644 --- a/drivers/firmware/arm_scmi/driver.c +++ b/drivers/firmware/arm_scmi/driver.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -2628,21 +2629,31 @@ static int scmi_handle_put(const struct scmi_handle *handle) return 0; } -static void scmi_device_link_add(struct device *consumer, +static bool scmi_device_link_add(struct device *consumer, struct device *supplier) { struct device_link *link; link = device_link_add(consumer, supplier, DL_FLAG_AUTOREMOVE_CONSUMER); - WARN_ON(!link); + return !WARN_ON(!link); +} + +static void scmi_clear_handle(struct scmi_device *scmi_dev) +{ + if (!scmi_dev->handle) + return; + + scmi_handle_put(scmi_dev->handle); + scmi_dev->handle = NULL; } static void scmi_set_handle(struct scmi_device *scmi_dev) { scmi_dev->handle = scmi_handle_get(&scmi_dev->dev); - if (scmi_dev->handle) - scmi_device_link_add(&scmi_dev->dev, scmi_dev->handle->dev); + if (scmi_dev->handle && + !scmi_device_link_add(&scmi_dev->dev, scmi_dev->handle->dev)) + scmi_clear_handle(scmi_dev); } static int __scmi_xfer_info_init(struct scmi_info *sinfo, @@ -2751,6 +2762,9 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node, idx = tx ? 0 : 1; idr = tx ? &info->tx_idr : &info->rx_idr; + if (idr_find(idr, prot_id)) + return -EEXIST; + if (!info->desc->ops->chan_available(of_node, idx)) { cinfo = idr_find(idr, SCMI_PROTOCOL_BASE); if (unlikely(!cinfo)) /* Possible only if platform has no Rx */ @@ -2768,7 +2782,7 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node, cinfo->no_completion_irq = info->desc->no_completion_irq; /* Create a unique name for this transport device */ - snprintf(name, 32, "__scmi_transport_device_%s_%02X", + snprintf(name, sizeof(name), SCMI_TRANSPORT_DEVNAME_PREFIX "_%s_%02X", idx ? "rx" : "tx", prot_id); /* Create a uniquely named, dedicated transport device for this chan */ tdev = scmi_device_create(of_node, info->dev, prot_id, name); @@ -2778,13 +2792,12 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node, devm_kfree(info->dev, cinfo); return -EINVAL; } - of_node_get(of_node); cinfo->id = prot_id; cinfo->dev = &tdev->dev; + cinfo->handle = &info->handle; ret = info->desc->ops->chan_setup(cinfo, info->dev, tx); if (ret) { - of_node_put(of_node); scmi_device_destroy(info->dev, prot_id, name); devm_kfree(info->dev, cinfo); return ret; @@ -2807,14 +2820,13 @@ idr_alloc: "unable to allocate SCMI idr slot err %d\n", ret); /* Destroy channel and device only if created by this call. */ if (tdev) { - of_node_put(of_node); + info->desc->ops->chan_free(prot_id, cinfo, idr); scmi_device_destroy(info->dev, prot_id, name); devm_kfree(info->dev, cinfo); } return ret; } - cinfo->handle = &info->handle; return 0; } @@ -2872,9 +2884,11 @@ static int scmi_channels_setup(struct scmi_info *info) if (of_property_read_u32(child, "reg", &prot_id)) continue; - if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) + if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) { dev_err(info->dev, "Out of range protocol %d\n", prot_id); + continue; + } ret = scmi_txrx_setup(info, child, prot_id); if (ret) @@ -2884,7 +2898,7 @@ static int scmi_channels_setup(struct scmi_info *info) return 0; } -static int scmi_chan_destroy(int id, void *p, void *idr) +static int scmi_chan_destroy(int id, void *p, void *data) { struct scmi_chan_info *cinfo = p; @@ -2892,13 +2906,10 @@ static int scmi_chan_destroy(int id, void *p, void *idr) struct scmi_info *info = handle_to_scmi_info(cinfo->handle); struct scmi_device *sdev = to_scmi_dev(cinfo->dev); - of_node_put(cinfo->dev->of_node); - scmi_device_destroy(info->dev, id, sdev->name); + scmi_device_destroy(info->dev, cinfo->id, sdev->name); cinfo->dev = NULL; } - idr_remove(idr, id); - return 0; } @@ -2925,6 +2936,7 @@ static int scmi_bus_notifier(struct notifier_block *nb, { struct scmi_info *info = bus_nb_to_scmi_info(nb); struct scmi_device *sdev = to_scmi_dev(data); + const char *status; /* Skip devices of different SCMI instances */ if (sdev->dev.parent != info->dev) @@ -2934,18 +2946,22 @@ static int scmi_bus_notifier(struct notifier_block *nb, case BUS_NOTIFY_BIND_DRIVER: /* setup handle now as the transport is ready */ scmi_set_handle(sdev); + status = "about to be BOUND."; + break; + case BUS_NOTIFY_DRIVER_NOT_BOUND: + scmi_clear_handle(sdev); + status = "NOT BOUND."; break; case BUS_NOTIFY_UNBOUND_DRIVER: - scmi_handle_put(sdev->handle); - sdev->handle = NULL; + scmi_clear_handle(sdev); + status = "UNBOUND."; break; default: return NOTIFY_DONE; } dev_dbg(info->dev, "Device %s (%s) is now %s\n", dev_name(&sdev->dev), - sdev->name, action == BUS_NOTIFY_BIND_DRIVER ? - "about to be BOUND." : "UNBOUND."); + sdev->name, status); return NOTIFY_OK; } @@ -2957,7 +2973,9 @@ static int scmi_device_request_notifier(struct notifier_block *nb, struct scmi_device_id *id_table = data; struct scmi_info *info = req_nb_to_scmi_info(nb); + rcu_read_lock(); np = idr_find(&info->active_protocols, id_table->protocol_id); + rcu_read_unlock(); if (!np) return NOTIFY_DONE; @@ -3263,7 +3281,7 @@ static int scmi_probe(struct platform_device *pdev) ret = scmi_channels_setup(info); if (ret) { err_str = "failed to setup channels\n"; - goto clear_ida; + goto clear_txrx_setup; } ret = bus_register_notifier(&scmi_bus_type, &info->bus_nb); @@ -3325,7 +3343,7 @@ static int scmi_probe(struct platform_device *pdev) dev_err(dev, "%s", err_str); return 0; } - goto notification_exit; + goto raw_mode_cleanup; } mutex_lock(&scmi_list_mutex); @@ -3340,8 +3358,10 @@ static int scmi_probe(struct platform_device *pdev) if (of_property_read_u32(child, "reg", &prot_id)) continue; - if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) + if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) { dev_err(dev, "Out of range protocol %d\n", prot_id); + continue; + } if (!scmi_is_protocol_implemented(handle, prot_id)) { dev_err(dev, "SCMI protocol %d not implemented\n", @@ -3367,18 +3387,18 @@ static int scmi_probe(struct platform_device *pdev) return 0; -notification_exit: +raw_mode_cleanup: if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT)) scmi_raw_mode_cleanup(info->raw); - scmi_notification_exit(&info->handle); clear_dev_req_notifier: blocking_notifier_chain_unregister(&scmi_requested_devices_nh, &info->dev_req_nb); clear_bus_notifier: bus_unregister_notifier(&scmi_bus_type, &info->bus_nb); clear_txrx_setup: + scmi_notification_quiesce(&info->handle); scmi_cleanup_txrx_channels(info); -clear_ida: + scmi_notification_exit(&info->handle); ida_free(&scmi_id, info->id); out_err: @@ -3401,6 +3421,12 @@ static void scmi_remove(struct platform_device *pdev) list_del(&info->node); mutex_unlock(&scmi_list_mutex); + blocking_notifier_chain_unregister(&scmi_requested_devices_nh, + &info->dev_req_nb); + + /* Stop transport callbacks before tearing down notifications. */ + scmi_notification_quiesce(&info->handle); + scmi_cleanup_txrx_channels(info); scmi_notification_exit(&info->handle); mutex_lock(&info->protocols_mtx); @@ -3411,13 +3437,8 @@ static void scmi_remove(struct platform_device *pdev) of_node_put(child); idr_destroy(&info->active_protocols); - blocking_notifier_chain_unregister(&scmi_requested_devices_nh, - &info->dev_req_nb); bus_unregister_notifier(&scmi_bus_type, &info->bus_nb); - /* Safe to free channels since no more users */ - scmi_cleanup_txrx_channels(info); - ida_free(&scmi_id, info->id); } diff --git a/drivers/firmware/arm_scmi/notify.c b/drivers/firmware/arm_scmi/notify.c index 0a192cf2deab62..dfe2aa89c50040 100644 --- a/drivers/firmware/arm_scmi/notify.c +++ b/drivers/firmware/arm_scmi/notify.c @@ -1706,6 +1706,25 @@ err: return -ENOMEM; } +/** + * scmi_notification_quiesce() - Stop notification late initialization + * @handle: The handle identifying the platform instance to quiesce + * + * Prevent new late-init work from being queued and wait for any already queued + * or running late-init work to complete before transport channels are torn + * down. + */ +void scmi_notification_quiesce(struct scmi_handle *handle) +{ + struct scmi_notify_instance *ni; + + ni = scmi_notification_instance_data_get(handle); + if (!ni) + return; + + disable_work_sync(&ni->init_work); +} + /** * scmi_notification_exit() - Shutdown and clean Notification core * @handle: The handle identifying the platform instance to shutdown @@ -1717,6 +1736,8 @@ void scmi_notification_exit(struct scmi_handle *handle) ni = scmi_notification_instance_data_get(handle); if (!ni) return; + + scmi_notification_quiesce(handle); scmi_notification_instance_data_set(handle, NULL); /* Destroy while letting pending work complete */ diff --git a/drivers/firmware/arm_scmi/notify.h b/drivers/firmware/arm_scmi/notify.h index 76758a736cf47a..f18f98c5ab3bad 100644 --- a/drivers/firmware/arm_scmi/notify.h +++ b/drivers/firmware/arm_scmi/notify.h @@ -82,6 +82,7 @@ struct scmi_protocol_events { }; int scmi_notification_init(struct scmi_handle *handle); +void scmi_notification_quiesce(struct scmi_handle *handle); void scmi_notification_exit(struct scmi_handle *handle); int scmi_register_protocol_events(const struct scmi_handle *handle, u8 proto_id, const struct scmi_protocol_handle *ph, diff --git a/drivers/firmware/arm_scmi/transports/mailbox.c b/drivers/firmware/arm_scmi/transports/mailbox.c index ae0f67e6cc45fe..308736c3ead9c5 100644 --- a/drivers/firmware/arm_scmi/transports/mailbox.c +++ b/drivers/firmware/arm_scmi/transports/mailbox.c @@ -211,13 +211,18 @@ static int mailbox_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, cl->tx_block = false; cl->knows_txdone = tx; + cinfo->transport_info = smbox; + smbox->cinfo = cinfo; + mutex_init(&smbox->chan_lock); + smbox->chan = mbox_request_channel(cl, tx ? 0 : p2a_chan); if (IS_ERR(smbox->chan)) { ret = PTR_ERR(smbox->chan); + smbox->chan = NULL; if (ret != -EPROBE_DEFER) dev_err(cdev, "failed to request SCMI %s mailbox\n", desc); - return ret; + goto err_clear_cinfo; } /* Additional unidirectional channel for TX if needed */ @@ -225,9 +230,10 @@ static int mailbox_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, smbox->chan_receiver = mbox_request_channel(cl, a2p_rx_chan); if (IS_ERR(smbox->chan_receiver)) { ret = PTR_ERR(smbox->chan_receiver); + smbox->chan_receiver = NULL; if (ret != -EPROBE_DEFER) dev_err(cdev, "failed to request SCMI Tx Receiver mailbox\n"); - return ret; + goto err_free_chan; } } @@ -235,17 +241,23 @@ static int mailbox_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, smbox->chan_platform_receiver = mbox_request_channel(cl, p2a_rx_chan); if (IS_ERR(smbox->chan_platform_receiver)) { ret = PTR_ERR(smbox->chan_platform_receiver); + smbox->chan_platform_receiver = NULL; if (ret != -EPROBE_DEFER) dev_err(cdev, "failed to request SCMI P2A Receiver mailbox\n"); - return ret; + goto err_free_chan; } } - cinfo->transport_info = smbox; - smbox->cinfo = cinfo; - mutex_init(&smbox->chan_lock); - return 0; + +err_free_chan: + mbox_free_channel(smbox->chan); +err_clear_cinfo: + cinfo->transport_info = NULL; + smbox->cinfo = NULL; + devm_iounmap(dev, smbox->shmem); + devm_kfree(dev, smbox); + return ret; } static int mailbox_chan_free(int id, void *p, void *data) diff --git a/drivers/firmware/arm_scmi/transports/smc.c b/drivers/firmware/arm_scmi/transports/smc.c index 21abb571e4f2f1..1fce3ccdeb7fcb 100644 --- a/drivers/firmware/arm_scmi/transports/smc.c +++ b/drivers/firmware/arm_scmi/transports/smc.c @@ -172,6 +172,13 @@ static int smc_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, scmi_info->param_page = SHMEM_PAGE(res.start); scmi_info->param_offset = SHMEM_OFFSET(res.start); } + + scmi_info->func_id = func_id; + scmi_info->cap_id = cap_id; + scmi_info->cinfo = cinfo; + smc_channel_lock_init(scmi_info); + cinfo->transport_info = scmi_info; + /* * If there is an interrupt named "a2p", then the service and * completion of a message is signaled by an interrupt rather than by @@ -183,18 +190,14 @@ static int smc_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, IRQF_NO_SUSPEND, dev_name(dev), scmi_info); if (ret) { dev_err(dev, "failed to setup SCMI smc irq\n"); + cinfo->transport_info = NULL; + scmi_info->cinfo = NULL; return ret; } } else { cinfo->no_completion_irq = true; } - scmi_info->func_id = func_id; - scmi_info->cap_id = cap_id; - scmi_info->cinfo = cinfo; - smc_channel_lock_init(scmi_info); - cinfo->transport_info = scmi_info; - return 0; } diff --git a/drivers/firmware/google/coreboot_table.c b/drivers/firmware/google/coreboot_table.c index e63933ff674700..96e68ae3f6accc 100644 --- a/drivers/firmware/google/coreboot_table.c +++ b/drivers/firmware/google/coreboot_table.c @@ -170,6 +170,7 @@ static int coreboot_table_populate(struct device *dev, void *ptr, resource_size_ static int coreboot_table_probe(struct platform_device *pdev) { resource_size_t len; + resource_size_t table_span; struct coreboot_table_header *header; struct resource *res; struct device *dev = &pdev->dev; @@ -181,7 +182,7 @@ static int coreboot_table_probe(struct platform_device *pdev) return -EINVAL; len = resource_size(res); - if (!res->start || !len) + if (!res->start || len < sizeof(*header)) return -EINVAL; /* Check just the header first to make sure things are sane */ @@ -189,19 +190,27 @@ static int coreboot_table_probe(struct platform_device *pdev) if (!header) return -ENOMEM; - len = header->header_bytes + header->table_bytes; ret = strncmp(header->signature, "LBIO", sizeof(header->signature)); + + if (!ret && + (header->header_bytes < sizeof(*header) || + check_add_overflow((resource_size_t)header->header_bytes, + (resource_size_t)header->table_bytes, + &table_span) || + table_span > len)) + ret = -EINVAL; + memunmap(header); if (ret) { dev_warn(dev, "coreboot table missing or corrupt!\n"); return -ENODEV; } - ptr = memremap(res->start, len, MEMREMAP_WB); + ptr = memremap(res->start, table_span, MEMREMAP_WB); if (!ptr) return -ENOMEM; - ret = coreboot_table_populate(dev, ptr, len); + ret = coreboot_table_populate(dev, ptr, table_span); memunmap(ptr); diff --git a/drivers/firmware/imx/sm-misc.c b/drivers/firmware/imx/sm-misc.c index ac9af824c2d409..fb8d7bdb5b0844 100644 --- a/drivers/firmware/imx/sm-misc.c +++ b/drivers/firmware/imx/sm-misc.c @@ -79,6 +79,9 @@ static int syslog_show(struct seq_file *file, void *priv) u16 size = SZ_4K / 4; int ret; + if (!syslog) + return -ENOMEM; + if (!ph) return -ENODEV; diff --git a/drivers/firmware/qcom/Makefile b/drivers/firmware/qcom/Makefile index 0be40a1abc13c3..b679d3fc2c267a 100644 --- a/drivers/firmware/qcom/Makefile +++ b/drivers/firmware/qcom/Makefile @@ -5,6 +5,7 @@ obj-$(CONFIG_QCOM_SCM) += qcom-scm.o qcom-scm-objs += qcom_scm.o qcom_scm-smc.o qcom_scm-legacy.o +CFLAGS_qcom_scm-smc.o := -I$(src) obj-$(CONFIG_QCOM_TZMEM) += qcom_tzmem.o obj-$(CONFIG_QCOM_QSEECOM) += qcom_qseecom.o obj-$(CONFIG_QCOM_QSEECOM_UEFISECAPP) += qcom_qseecom_uefisecapp.o diff --git a/drivers/firmware/qcom/qcom_scm-smc.c b/drivers/firmware/qcom/qcom_scm-smc.c index 574930729ddd72..127365ab11fc24 100644 --- a/drivers/firmware/qcom/qcom_scm-smc.c +++ b/drivers/firmware/qcom/qcom_scm-smc.c @@ -24,6 +24,9 @@ struct arm_smccc_args { unsigned long args[8]; }; +#define CREATE_TRACE_POINTS +#include "qcom_scm_trace.h" + static DEFINE_MUTEX(qcom_scm_lock); #define QCOM_SCM_EBUSY_WAIT_MS 30 @@ -44,6 +47,7 @@ static void __scm_smc_do_quirk(const struct arm_smccc_args *smc, quirk.state.a6 = 0; do { + trace_scm_smc_request(a0, smc); arm_smccc_smc_quirk(a0, smc->args[1], smc->args[2], smc->args[3], smc->args[4], smc->args[5], quirk.state.a6, smc->args[7], res, &quirk); @@ -83,6 +87,7 @@ int scm_get_wq_ctx(u32 *wq_ctx, u32 *flags, u32 *more_pending) if (ret) return ret; + trace_scm_waitq_get_wq_ctx(get_wq_res.a1, get_wq_res.a2, get_wq_res.a3); *wq_ctx = get_wq_res.a1; *flags = get_wq_res.a2; *more_pending = get_wq_res.a3; @@ -105,10 +110,12 @@ static int __scm_smc_do_quirk_handle_waitq(struct device *dev, struct arm_smccc_ wq_ctx = res->a1; smc_call_ctx = res->a2; - ret = qcom_scm_wait_for_wq_completion(wq_ctx); + trace_scm_waitq_sleep(wq_ctx, smc_call_ctx); + ret = qcom_scm_wait_for_wq_completion(dev, wq_ctx); if (ret) return ret; + trace_scm_waitq_resume(smc_call_ctx); fill_wq_resume_args(&resume, smc_call_ctx); smc = &resume; } @@ -201,6 +208,9 @@ int __scm_smc_call(struct device *dev, const struct qcom_scm_desc *desc, } ret = __scm_smc_do(dev, &smc, &smc_res, atomic); + + trace_scm_smc_done(ret, smc.args[0], &smc_res); + if (ret) return ret; diff --git a/drivers/firmware/qcom/qcom_scm.c b/drivers/firmware/qcom/qcom_scm.c index 6b601a4b89dbf0..26bf87247afa10 100644 --- a/drivers/firmware/qcom/qcom_scm.c +++ b/drivers/firmware/qcom/qcom_scm.c @@ -2630,23 +2630,20 @@ static int qcom_scm_get_waitq_irq(struct qcom_scm *scm) return irq_create_fwspec_mapping(&fwspec); } -static struct completion *qcom_scm_get_completion(u32 wq_ctx) +static struct completion *qcom_scm_get_completion(struct qcom_scm *scm, u32 wq_ctx) { - struct completion *wq; - - if (WARN_ON_ONCE(wq_ctx >= __scm->wq_cnt)) + if (WARN_ON_ONCE(wq_ctx >= scm->wq_cnt)) return ERR_PTR(-EINVAL); - wq = &__scm->waitq_comps[wq_ctx]; - - return wq; + return &scm->waitq_comps[wq_ctx]; } -int qcom_scm_wait_for_wq_completion(u32 wq_ctx) +int qcom_scm_wait_for_wq_completion(struct device *dev, u32 wq_ctx) { + struct qcom_scm *scm = dev_get_drvdata(dev); struct completion *wq; - wq = qcom_scm_get_completion(wq_ctx); + wq = qcom_scm_get_completion(scm, wq_ctx); if (IS_ERR(wq)) return PTR_ERR(wq); @@ -2655,11 +2652,11 @@ int qcom_scm_wait_for_wq_completion(u32 wq_ctx) return 0; } -static int qcom_scm_waitq_wakeup(unsigned int wq_ctx) +static int qcom_scm_waitq_wakeup(struct qcom_scm *scm, unsigned int wq_ctx) { struct completion *wq; - wq = qcom_scm_get_completion(wq_ctx); + wq = qcom_scm_get_completion(scm, wq_ctx); if (IS_ERR(wq)) return PTR_ERR(wq); @@ -2686,7 +2683,7 @@ static irqreturn_t qcom_scm_irq_handler(int irq, void *data) goto out; } - ret = qcom_scm_waitq_wakeup(wq_ctx); + ret = qcom_scm_waitq_wakeup(scm, wq_ctx); if (ret) goto out; } while (more_pending); @@ -2746,6 +2743,7 @@ static int qcom_scm_probe(struct platform_device *pdev) return -ENOMEM; scm->dev = &pdev->dev; + platform_set_drvdata(pdev, scm); ret = qcom_scm_find_dload_address(&pdev->dev, &scm->dload_mode_addr); if (ret < 0) return ret; @@ -2787,9 +2785,11 @@ static int qcom_scm_probe(struct platform_device *pdev) "Failed to setup the reserved memory region for TZ mem\n"); ret = qcom_tzmem_enable(scm->dev); - if (ret) - return dev_err_probe(scm->dev, ret, - "Failed to enable the TrustZone memory allocator\n"); + if (ret) { + ret = dev_err_probe(scm->dev, ret, + "Failed to enable the TrustZone memory allocator\n"); + goto err_rmem; + } memset(&pool_config, 0, sizeof(pool_config)); pool_config.initial_size = 0; @@ -2797,9 +2797,11 @@ static int qcom_scm_probe(struct platform_device *pdev) pool_config.max_size = SZ_256K; scm->mempool = devm_qcom_tzmem_pool_new(scm->dev, &pool_config); - if (IS_ERR(scm->mempool)) - return dev_err_probe(scm->dev, PTR_ERR(scm->mempool), - "Failed to create the SCM memory pool\n"); + if (IS_ERR(scm->mempool)) { + ret = dev_err_probe(scm->dev, PTR_ERR(scm->mempool), + "Failed to create the SCM memory pool\n"); + goto err_rmem; + } ret = qcom_scm_query_waitq_count(scm); scm->wq_cnt = ret < 0 ? QCOM_SCM_DEFAULT_WAITQ_COUNT : ret; @@ -2870,6 +2872,10 @@ static int qcom_scm_probe(struct platform_device *pdev) qcom_scm_gunyah_wdt_init(scm); return 0; + +err_rmem: + of_reserved_mem_device_release(scm->dev); + return ret; } static void qcom_scm_shutdown(struct platform_device *pdev) diff --git a/drivers/firmware/qcom/qcom_scm.h b/drivers/firmware/qcom/qcom_scm.h index caab80a73e17f5..cf90a565fdfbd7 100644 --- a/drivers/firmware/qcom/qcom_scm.h +++ b/drivers/firmware/qcom/qcom_scm.h @@ -66,7 +66,7 @@ struct qcom_scm_res { u64 result[MAX_QCOM_SCM_RETS]; }; -int qcom_scm_wait_for_wq_completion(u32 wq_ctx); +int qcom_scm_wait_for_wq_completion(struct device *dev, u32 wq_ctx); int scm_get_wq_ctx(u32 *wq_ctx, u32 *flags, u32 *more_pending); #define SCM_SMC_FNID(s, c) ((((s) & 0xFF) << 8) | ((c) & 0xFF)) diff --git a/drivers/firmware/qcom/qcom_scm_trace.h b/drivers/firmware/qcom/qcom_scm_trace.h new file mode 100644 index 00000000000000..6c911124fc56b6 --- /dev/null +++ b/drivers/firmware/qcom/qcom_scm_trace.h @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM qcom_scm + +#if !defined(_TRACE_SCM_SMC_INTERFACE_H) || defined(TRACE_HEADER_MULTI_READ) + +#define _TRACE_SCM_SMC_INTERFACE_H + +#include + +TRACE_EVENT(scm_smc_request, + + TP_PROTO(unsigned long a0, const struct arm_smccc_args *smc), + + TP_ARGS(a0, smc), + + TP_STRUCT__entry( + __field(u64, smc_id) + __field(u8, svc_id) + __field(u8, cmd_id) + __field(u8, args_cnt) + __dynamic_array(unsigned long, args, + min_t(u8, (smc->args[1] & 0xF), (u8)6)) + ), + + TP_fast_assign( + __entry->smc_id = a0; + __entry->svc_id = (smc->args[0] >> 8) & 0xFF; + __entry->cmd_id = smc->args[0] & 0xFF; + u8 n = min_t(u8, (smc->args[1] & 0xF), (u8)6); + + __entry->args_cnt = n; + + unsigned long *dst = __get_dynamic_array(args); + + for (int i = 0; i < n; i++) + dst[i] = smc->args[2 + i]; + ), + + TP_printk("smc_id:0x%08llx svc_id:0x%02x cmd_id:0x%02x args_cnt:%u args:%s", + __entry->smc_id, __entry->svc_id, __entry->cmd_id, __entry->args_cnt, + __print_dynamic_array(args, sizeof(unsigned long))) +); + +TRACE_EVENT(scm_waitq_sleep, + + TP_PROTO(u32 wq_ctx, u32 smc_ctx), + + TP_ARGS(wq_ctx, smc_ctx), + + TP_STRUCT__entry( + __field(u32, wq_ctx) + __field(u32, smc_call_ctx) + ), + + TP_fast_assign( + __entry->wq_ctx = wq_ctx; + __entry->smc_call_ctx = smc_ctx; + ), + + TP_printk("wq_ctx:%u, smc_call_ctx:%u", __entry->wq_ctx, __entry->smc_call_ctx) +); + +TRACE_EVENT(scm_waitq_resume, + + TP_PROTO(u32 smc_ctx), + + TP_ARGS(smc_ctx), + + TP_STRUCT__entry( + __field(u32, smc_call_ctx) + ), + + TP_fast_assign( + __entry->smc_call_ctx = smc_ctx; + ), + + TP_printk("smc_call_ctx:%u", __entry->smc_call_ctx) +); + +TRACE_EVENT(scm_waitq_get_wq_ctx, + + TP_PROTO(u32 wq_ctx, u32 flags, u32 pending), + + TP_ARGS(wq_ctx, flags, pending), + + TP_STRUCT__entry( + __field(u32, wq_ctx) + __field(u32, flags) + __field(u32, more_pending) + ), + + TP_fast_assign( + __entry->wq_ctx = wq_ctx; + __entry->flags = flags; + __entry->more_pending = pending; + ), + + TP_printk("wq_ctx:%u, flags:%u, more_pending:%u", + __entry->wq_ctx, __entry->flags, __entry->more_pending) +); + +TRACE_EVENT(scm_smc_done, + + TP_PROTO(int ret, u64 smc_id, struct arm_smccc_res *smc_res), + + TP_ARGS(ret, smc_id, smc_res), + + TP_STRUCT__entry( + __field(int, ret) + __field(u64, smc_id) + __field(unsigned long, res) + __field(unsigned long, res0) + __field(unsigned long, res1) + __field(unsigned long, res2) + ), + + TP_fast_assign( + __entry->ret = ret; + __entry->smc_id = smc_id; + __entry->res = smc_res->a0; + __entry->res0 = smc_res->a1; + __entry->res1 = smc_res->a2; + __entry->res2 = smc_res->a3; + ), + + TP_printk("smc_id:0x%08llx, ret:%d res_to_callee:0x%lx res0:0x%lx res1:0x%lx res2:0x%lx", + __entry->smc_id, __entry->ret, __entry->res, + __entry->res0, __entry->res1, __entry->res2) +); + +#endif /* _TRACE_SCM_SMC_INTERFACE_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE qcom_scm_trace + +#include + diff --git a/drivers/firmware/qcom/qcom_tzmem.c b/drivers/firmware/qcom/qcom_tzmem.c index 0635cbeacfc8e6..0fd9581275f173 100644 --- a/drivers/firmware/qcom/qcom_tzmem.c +++ b/drivers/firmware/qcom/qcom_tzmem.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -507,14 +508,18 @@ phys_addr_t qcom_tzmem_to_phys(void *vaddr) } EXPORT_SYMBOL_GPL(qcom_tzmem_to_phys); +static void qcom_tzmem_do_init(int *result) +{ + *result = qcom_tzmem_init(); +} + int qcom_tzmem_enable(struct device *dev) { - if (qcom_tzmem_dev) - return -EBUSY; + static int result; qcom_tzmem_dev = dev; - - return qcom_tzmem_init(); + DO_ONCE(qcom_tzmem_do_init, &result); + return result; } EXPORT_SYMBOL_GPL(qcom_tzmem_enable); diff --git a/drivers/gpib/common/gpib_os.c b/drivers/gpib/common/gpib_os.c index 69f6aa73ab9a3c..384800e6bf975a 100644 --- a/drivers/gpib/common/gpib_os.c +++ b/drivers/gpib/common/gpib_os.c @@ -289,18 +289,19 @@ int autopoll_all_devices(struct gpib_board *board) dev_dbg(board->gpib_dev, "autopoll has board lock\n"); retval = serial_poll_all(board, serial_timeout); - if (retval < 0) { - mutex_unlock(&board->big_gpib_mutex); - mutex_unlock(&board->user_mutex); - return retval; + if (retval >= 0) { + dev_dbg(board->gpib_dev, "complete\n"); + /* + * need to wake wait queue in case someone is + * waiting on RQS + */ + wake_up_interruptible(&board->wait); } - dev_dbg(board->gpib_dev, "complete\n"); - /* - * need to wake wait queue in case someone is - * waiting on RQS - */ - wake_up_interruptible(&board->wait); + if (retval <= 0) { + atomic_set(&board->stuck_srq, 1); + set_bit(SRQI_NUM, &board->status); + } mutex_unlock(&board->big_gpib_mutex); mutex_unlock(&board->user_mutex); diff --git a/drivers/gpib/common/iblib.c b/drivers/gpib/common/iblib.c index b672dd6aad25f2..511e1d61c1fb24 100644 --- a/drivers/gpib/common/iblib.c +++ b/drivers/gpib/common/iblib.c @@ -193,9 +193,6 @@ static int autospoll_thread(void *board_void) } if (retval <= 0) { dev_err(board->gpib_dev, "stuck SRQ\n"); - - atomic_set(&board->stuck_srq, 1); // XXX could be better - set_bit(SRQI_NUM, &board->status); } } return retval; diff --git a/drivers/gpib/include/amccs5933.h b/drivers/gpib/include/amccs5933.h index d7f63c7950963a..f8a80bdc70dcce 100644 --- a/drivers/gpib/include/amccs5933.h +++ b/drivers/gpib/include/amccs5933.h @@ -13,7 +13,7 @@ enum { }; // incoming mailbox 0-3 register offsets -extern inline int INCOMING_MAILBOX_REG(unsigned int mailbox) +static inline int INCOMING_MAILBOX_REG(unsigned int mailbox) { return (0x10 + 4 * mailbox); }; @@ -29,25 +29,25 @@ enum { }; // select byte 0 to 3 of incoming mailbox -extern inline int INBOX_BYTE_BITS(unsigned int byte) +static inline int INBOX_BYTE_BITS(unsigned int byte) { return (byte & 0x3) << 8; }; // select incoming mailbox 0 to 3 -extern inline int INBOX_SELECT_BITS(unsigned int mailbox) +static inline int INBOX_SELECT_BITS(unsigned int mailbox) { return (mailbox & 0x3) << 10; }; // select byte 0 to 3 of outgoing mailbox -extern inline int OUTBOX_BYTE_BITS(unsigned int byte) +static inline int OUTBOX_BYTE_BITS(unsigned int byte) { return (byte & 0x3); }; // select outgoing mailbox 0 to 3 -extern inline int OUTBOX_SELECT_BITS(unsigned int mailbox) +static inline int OUTBOX_SELECT_BITS(unsigned int mailbox) { return (mailbox & 0x3) << 2; }; diff --git a/drivers/gpib/tnt4882/mite.h b/drivers/gpib/tnt4882/mite.h index a1fdba9672a038..dd251afa90e342 100644 --- a/drivers/gpib/tnt4882/mite.h +++ b/drivers/gpib/tnt4882/mite.h @@ -45,12 +45,12 @@ struct mite_struct { extern struct mite_struct *mite_devices; -extern inline unsigned int mite_irq(struct mite_struct *mite) +static inline unsigned int mite_irq(struct mite_struct *mite) { return mite->pcidev->irq; }; -extern inline unsigned int mite_device_id(struct mite_struct *mite) +static inline unsigned int mite_device_id(struct mite_struct *mite) { return mite->pcidev->device; }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c index e3972673fd6416..c47e5ffd3d0c71 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c @@ -237,7 +237,7 @@ int amdgpu_mes_init(struct amdgpu_device *adev) } if (adev->mes.hung_queue_db_array_size) { - for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) { + for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) { r = amdgpu_bo_create_kernel(adev, adev->mes.hung_queue_db_array_size * sizeof(u32), PAGE_SIZE, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c index e418390353b1b7..e55c9002113c76 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c @@ -1823,6 +1823,21 @@ static void gfx_v12_0_constants_init(struct amdgpu_device *adev) gfx_v12_0_get_tcc_info(adev); adev->gfx.config.pa_sc_tile_steering_override = 0; + /* Set whether texture coordinate truncation is conformant. */ + tmp = RREG32_SOC15(GC, 0, regTA_CNTL2); + adev->gfx.config.ta_cntl2_truncate_coord_mode = + REG_GET_FIELD(tmp, TA_CNTL2, TRUNCATE_COORD_MODE); + + /* Program DB_RING_CONTROL for multiple GFX pipes + * Default power up value is 1. + * Possible values: + * 0 - split occlusion counters between gfx pipes + * 1 - all occlusion counters to pipe 0 + * 2 - all occlusion counters to pipe 1 + */ + WREG32_FIELD15_PREREG(GC, 0, DB_RING_CONTROL, COUNTER_CONTROL, + (adev->gfx.me.num_pipe_per_me > 1) ? 0 : 1); + /* Set whether texture coordinate truncation is conformant. */ tmp = RREG32_SOC15(GC, 0, regTA_CNTL2); adev->gfx.config.ta_cntl2_truncate_coord_mode = diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c index ac90d8e9d86a83..8f3f90d5ce375d 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c @@ -1881,11 +1881,13 @@ static int gfx_v6_0_ring_test_ring(struct amdgpu_ring *ring) return r; } -static void gfx_v6_0_ring_emit_vgt_flush(struct amdgpu_ring *ring) +static void gfx_v6_0_ring_emit_event_write(struct amdgpu_ring *ring, + uint32_t event_type, + uint32_t event_index) { amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0)); - amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) | - EVENT_INDEX(0)); + amdgpu_ring_write(ring, EVENT_TYPE(event_type) | + EVENT_INDEX(event_index)); } static void gfx_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, @@ -1924,12 +1926,6 @@ static void gfx_v6_0_ring_emit_ib(struct amdgpu_ring *ring, unsigned vmid = AMDGPU_JOB_GET_VMID(job); u32 header, control = 0; - /* insert SWITCH_BUFFER packet before first IB in the ring frame */ - if (flags & AMDGPU_HAVE_CTX_SWITCH) { - amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); - amdgpu_ring_write(ring, 0); - } - if (ib->flags & AMDGPU_IB_FLAG_CE) header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); else @@ -2351,7 +2347,7 @@ static int gfx_v6_0_cp_resume(struct amdgpu_device *adev) static void gfx_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) { - int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); + int usepfp = 1; uint32_t seq = ring->fence_drv.sync_seq; uint64_t addr = ring->fence_drv.gpu_addr; @@ -2364,20 +2360,12 @@ static void gfx_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) amdgpu_ring_write(ring, seq); amdgpu_ring_write(ring, 0xffffffff); amdgpu_ring_write(ring, 4); /* poll interval */ - - if (usepfp) { - /* synce CE with ME to prevent CE fetch CEIB before context switch done */ - amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); - amdgpu_ring_write(ring, 0); - amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); - amdgpu_ring_write(ring, 0); - } } static void gfx_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring, unsigned vmid, uint64_t pd_addr) { - int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); + int usepfp = 1; amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); @@ -2395,19 +2383,13 @@ static void gfx_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring, /* sync PFP to ME, otherwise we might get invalid PFP reads */ amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); amdgpu_ring_write(ring, 0x0); - - /* synce CE with ME to prevent CE fetch CEIB before context switch done */ - amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); - amdgpu_ring_write(ring, 0); - amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); - amdgpu_ring_write(ring, 0); } } static void gfx_v6_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val) { - int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); + int usepfp = 1; amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | @@ -2996,12 +2978,30 @@ static uint64_t gfx_v6_0_get_gpu_clock_counter(struct amdgpu_device *adev) return clock; } +static void gfx_v6_0_ring_emit_sb(struct amdgpu_ring *ring) +{ + amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); + amdgpu_ring_write(ring, 0); +} + static void gfx_v6_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) { - if (flags & AMDGPU_HAVE_CTX_SWITCH) - gfx_v6_0_ring_emit_vgt_flush(ring); + u32 dw2 = 0x80000000; /* set load_enable otherwise this package is just NOPs */ + + if (flags & AMDGPU_HAVE_CTX_SWITCH) { + gfx_v6_0_ring_emit_event_write(ring, VS_PARTIAL_FLUSH, 4); + gfx_v6_0_ring_emit_event_write(ring, VGT_FLUSH, 0); + + /* set load_global_config (load_global_uconfig doesn't exist on GFX6) */ + dw2 |= 0x1; + /* set load_cs_sh_regs */ + dw2 |= 0x01000000; + /* set load_per_context_state & load_gfx_sh_regs */ + dw2 |= 0x10002; + } + amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); - amdgpu_ring_write(ring, 0x80000000); + amdgpu_ring_write(ring, dw2); amdgpu_ring_write(ring, 0); } @@ -3527,11 +3527,12 @@ static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_gfx = { .emit_frame_size = 5 + 5 + /* hdp flush / invalidate */ 14 + 14 + 14 + /* gfx_v6_0_ring_emit_fence x3 for user fence, vm fence */ - 7 + 4 + /* gfx_v6_0_ring_emit_pipeline_sync */ - SI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + 6 + /* gfx_v6_0_ring_emit_vm_flush */ - 3 + 2 + /* gfx_v6_ring_emit_cntxcntl including vgt flush */ + 7 + /* gfx_v6_0_ring_emit_pipeline_sync */ + SI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + 2 + /* gfx_v6_0_ring_emit_vm_flush */ + 3 * 2 + /* gfx_v6_0_ring_emit_sb x3 (from amdgpu_vm_flush, amdgpu_ib_schedule) */ + 3 + 2 + 2 + /* gfx_v6_ring_emit_cntxcntl including VGT flush */ 5, /* SURFACE_SYNC */ - .emit_ib_size = 6, /* gfx_v6_0_ring_emit_ib */ + .emit_ib_size = 4, /* gfx_v6_0_ring_emit_ib */ .emit_ib = gfx_v6_0_ring_emit_ib, .emit_fence = gfx_v6_0_ring_emit_fence, .emit_pipeline_sync = gfx_v6_0_ring_emit_pipeline_sync, @@ -3539,6 +3540,7 @@ static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_gfx = { .test_ring = gfx_v6_0_ring_test_ring, .test_ib = gfx_v6_0_ring_test_ib, .insert_nop = amdgpu_ring_insert_nop, + .emit_switch_buffer = gfx_v6_0_ring_emit_sb, .emit_cntxcntl = gfx_v6_ring_emit_cntxcntl, .emit_wreg = gfx_v6_0_ring_emit_wreg, .emit_mem_sync = gfx_v6_0_emit_mem_sync, @@ -3554,10 +3556,11 @@ static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_compute = { .emit_frame_size = 5 + 5 + /* hdp flush / invalidate */ 7 + /* gfx_v6_0_ring_emit_pipeline_sync */ - SI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v6_0_ring_emit_vm_flush */ + SI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + 2 + /* gfx_v6_0_ring_emit_vm_flush */ 14 + 14 + 14 + /* gfx_v6_0_ring_emit_fence x3 for user fence, vm fence */ + 3 * 2 + /* gfx_v6_0_ring_emit_sb x3 (from amdgpu_vm_flush, amdgpu_ib_schedule) */ 5, /* SURFACE_SYNC */ - .emit_ib_size = 6, /* gfx_v6_0_ring_emit_ib */ + .emit_ib_size = 4, /* gfx_v6_0_ring_emit_ib */ .emit_ib = gfx_v6_0_ring_emit_ib, .emit_fence = gfx_v6_0_ring_emit_fence, .emit_pipeline_sync = gfx_v6_0_ring_emit_pipeline_sync, @@ -3565,6 +3568,7 @@ static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_compute = { .test_ring = gfx_v6_0_ring_test_ring, .test_ib = gfx_v6_0_ring_test_ib, .insert_nop = amdgpu_ring_insert_nop, + .emit_switch_buffer = gfx_v6_0_ring_emit_sb, .emit_wreg = gfx_v6_0_ring_emit_wreg, .emit_mem_sync = gfx_v6_0_emit_mem_sync, }; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index f059bed728f0ad..170c6b8d0a5f67 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -3284,17 +3284,26 @@ static int dm_hw_init(struct amdgpu_ip_block *ip_block) struct amdgpu_device *adev = ip_block->adev; int r; + adev->dm.i2c_devres_group = devres_open_group(adev->dev, NULL, GFP_KERNEL); + if (!adev->dm.i2c_devres_group) + return -ENOMEM; + /* Create DAL display manager */ r = amdgpu_dm_init(adev); if (r) - return r; + goto err_release_i2c; amdgpu_dm_hpd_init(adev); r = dm_oem_i2c_hw_init(adev); if (r) drm_info(adev_to_drm(adev), "Failed to add OEM i2c bus\n"); + devres_close_group(adev->dev, adev->dm.i2c_devres_group); return 0; + +err_release_i2c: + devres_release_group(adev->dev, adev->dm.i2c_devres_group); + return r; } /** @@ -3309,6 +3318,9 @@ static int dm_hw_fini(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; + if (adev->dm.i2c_devres_group) + devres_release_group(adev->dev, adev->dm.i2c_devres_group); + amdgpu_dm_hpd_fini(adev); amdgpu_dm_irq_fini(adev); @@ -5271,10 +5283,10 @@ static int get_brightness_range(const struct amdgpu_dm_backlight_caps *caps, return 1; } -/* Rescale from [min..max] to [0..AMDGPU_MAX_BL_LEVEL] */ -static inline u32 scale_input_to_fw(int min, int max, u64 input) +/* Rescale userspace [0..max] to the firmware curve's [0..255]. */ +static inline u32 scale_input_to_fw(int max, u64 input) { - return DIV_ROUND_CLOSEST_ULL(input * AMDGPU_MAX_BL_LEVEL, max - min); + return DIV_ROUND_CLOSEST_ULL(input * AMDGPU_MAX_BL_LEVEL, max); } /* Rescale from [0..AMDGPU_MAX_BL_LEVEL] to [min..max] */ @@ -5287,7 +5299,7 @@ static void convert_custom_brightness(const struct amdgpu_dm_backlight_caps *cap unsigned int min, unsigned int max, uint32_t *user_brightness) { - u32 brightness = scale_input_to_fw(min, max, *user_brightness); + u32 brightness = scale_input_to_fw(max, *user_brightness); u8 lower_signal, upper_signal, upper_lum, lower_lum, lum; int left, right; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h index dd199e0b792265..797f9447181082 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h @@ -688,6 +688,13 @@ struct amdgpu_display_manager { */ void *bb_from_dmub; + /** + * @i2c_devres_group: + * + * Devres group for DM i2c adapter lifetime management. + */ + void *i2c_devres_group; + /** * @oem_i2c: * diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c index 56bf907f1f6cc8..f47ee9937adaa0 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c @@ -464,13 +464,13 @@ static void amdgpu_dm_crtc_reset_state(struct drm_crtc *crtc) { struct dm_crtc_state *state; - if (crtc->state) - amdgpu_dm_crtc_destroy_state(crtc, crtc->state); - state = kzalloc_obj(*state); - if (WARN_ON(!state)) + if (!state) return; + if (crtc->state) + amdgpu_dm_crtc_destroy_state(crtc, crtc->state); + __drm_atomic_helper_crtc_reset(crtc, &state->base); } diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c index 8304e8897a3eda..acb38051c96e4c 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c @@ -294,7 +294,7 @@ void hdcp_reset_display(struct hdcp_workqueue *hdcp_work, unsigned int link_inde cancel_delayed_work(&hdcp_w->property_validate_dwork); - for (conn_index = 0; conn_index < AMDGPU_DM_MAX_DISPLAY_INDEX; conn_index++) { + for (conn_index = 0; conn_index < AMDGPU_DM_MAX_DISPLAY_COUNT; conn_index++) { hdcp_w->encryption_status[conn_index] = MOD_HDCP_ENCRYPTION_STATUS_HDCP_OFF; if (hdcp_w->aconnector[conn_index]) { @@ -341,7 +341,7 @@ static void event_property_update(struct work_struct *work) struct drm_connector *connector; struct drm_connector_state *conn_state; - for (conn_index = 0; conn_index < AMDGPU_DM_MAX_DISPLAY_INDEX; conn_index++) { + for (conn_index = 0; conn_index < AMDGPU_DM_MAX_DISPLAY_COUNT; conn_index++) { aconnector = hdcp_work->aconnector[conn_index]; if (!aconnector) @@ -410,7 +410,7 @@ static void event_property_validate(struct work_struct *work) guard(mutex)(&hdcp_work->mutex); - for (conn_index = 0; conn_index < AMDGPU_DM_MAX_DISPLAY_INDEX; + for (conn_index = 0; conn_index < AMDGPU_DM_MAX_DISPLAY_COUNT; conn_index++) { aconnector = hdcp_work->aconnector[conn_index]; @@ -817,10 +817,10 @@ struct hdcp_workqueue *hdcp_create_workqueue(struct amdgpu_device *adev, memset(hdcp_work[i].aconnector, 0, sizeof(struct amdgpu_dm_connector *) * - AMDGPU_DM_MAX_DISPLAY_INDEX); + AMDGPU_DM_MAX_DISPLAY_COUNT); memset(hdcp_work[i].encryption_status, 0, sizeof(enum mod_hdcp_encryption_status) * - AMDGPU_DM_MAX_DISPLAY_INDEX); + AMDGPU_DM_MAX_DISPLAY_COUNT); } cp_psp->funcs.update_stream_config = update_config; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h index 90b18c450ca61e..1c6436eebb063f 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h @@ -36,7 +36,7 @@ * Minimal declarations needed by this header. * Full amdgpu/DM definitions come from amdgpu_dm.h included by each .c file. */ -#define AMDGPU_DM_MAX_DISPLAY_INDEX 31 +#define AMDGPU_DM_MAX_DISPLAY_COUNT 32 struct amdgpu_dm_connector; struct mod_hdcp; @@ -51,7 +51,7 @@ struct hdcp_workqueue { struct delayed_work callback_dwork; struct delayed_work watchdog_timer_dwork; struct delayed_work property_validate_dwork; - struct amdgpu_dm_connector *aconnector[AMDGPU_DM_MAX_DISPLAY_INDEX]; + struct amdgpu_dm_connector *aconnector[AMDGPU_DM_MAX_DISPLAY_COUNT]; struct mutex mutex; struct mod_hdcp hdcp; @@ -59,7 +59,7 @@ struct hdcp_workqueue { struct mod_hdcp_display display; struct mod_hdcp_link link; - enum mod_hdcp_encryption_status encryption_status[AMDGPU_DM_MAX_DISPLAY_INDEX]; + enum mod_hdcp_encryption_status encryption_status[AMDGPU_DM_MAX_DISPLAY_COUNT]; /* when display is unplugged from mst hub, connctor will be * destroyed within dm_dp_mst_connector_destroy. connector * hdcp perperties, like type, undesired, desired, enabled, @@ -69,9 +69,9 @@ struct hdcp_workqueue { * will be retrieved from hdcp_work within dm_dp_mst_get_modes */ /* un-desired, desired, enabled */ - unsigned int content_protection[AMDGPU_DM_MAX_DISPLAY_INDEX]; + unsigned int content_protection[AMDGPU_DM_MAX_DISPLAY_COUNT]; /* hdcp1.x, hdcp2.x */ - unsigned int hdcp_content_type[AMDGPU_DM_MAX_DISPLAY_INDEX]; + unsigned int hdcp_content_type[AMDGPU_DM_MAX_DISPLAY_COUNT]; uint8_t max_link; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c index c7f8e08feaf4bf..cfd76c54f652bd 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c @@ -1488,17 +1488,15 @@ static const struct drm_plane_helper_funcs dm_primary_plane_helper_funcs = { static void amdgpu_dm_plane_drm_plane_reset(struct drm_plane *plane) { - struct dm_plane_state *amdgpu_state = NULL; - - if (plane->state) - plane->funcs->atomic_destroy_state(plane, plane->state); + struct dm_plane_state *amdgpu_state; amdgpu_state = kzalloc_obj(*amdgpu_state); - WARN_ON(amdgpu_state == NULL); - if (!amdgpu_state) return; + if (plane->state) + plane->funcs->atomic_destroy_state(plane, plane->state); + __drm_atomic_helper_plane_reset(plane, &amdgpu_state->base); amdgpu_state->degamma_tf = AMDGPU_TRANSFER_FUNCTION_DEFAULT; amdgpu_state->hdr_mult = AMDGPU_HDR_MULT_DEFAULT; diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c index 483cd9ab7eb768..42066b8a03623f 100644 --- a/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c +++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c @@ -572,7 +572,7 @@ static void dccg35_set_hdmistreamclk_src_new( case 0: REG_UPDATE_2(HDMISTREAMCLK_CNTL, HDMISTREAMCLK0_EN, (src == HDMI_STREAM_REFCLK) ? 0 : 1, - DPSTREAMCLK0_SRC_SEL, + HDMISTREAMCLK0_SRC_SEL, (src == HDMI_STREAM_REFCLK) ? 0 : src); break; default: diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c index 7c97a774141ff0..d8eb5996b57745 100644 --- a/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c +++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c @@ -772,8 +772,7 @@ bool hubp1_is_flip_pending(struct hubp *hubp) if (flip_pending) return true; - if (hubp && - earliest_inuse_address.grph.addr.quad_part != hubp->request_address.grph.addr.quad_part) + if (earliest_inuse_address.grph.addr.quad_part != hubp->request_address.grph.addr.quad_part) return true; return false; diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c index 54ce768ae6ad97..da679fb7d89c77 100644 --- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c +++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c @@ -39,7 +39,6 @@ #include "link/link_dpms.h" #include "dm_helpers.h" #include "link_dp_dpia_bw.h" -#include "link_dp_panel_replay.h" #define DC_LOGGER \ link->ctx->logger diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c index ce166a7f8e420c..1fff7567bca275 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c @@ -268,15 +268,21 @@ static const ATOM_VOLTAGE_OBJECT_V3 *atomctrl_lookup_voltage_type_v3( unsigned int offset = offsetof(ATOM_VOLTAGE_OBJECT_INFO_V3_1, asVoltageObj[0]); uint8_t *start = (uint8_t *)voltage_object_info_table; - while (offset < size) { + while (offset + sizeof(ATOM_VOLTAGE_OBJECT_HEADER_V3) <= size) { const ATOM_VOLTAGE_OBJECT_V3 *voltage_object = (const ATOM_VOLTAGE_OBJECT_V3 *)(start + offset); + u16 obj_size; + + obj_size = le16_to_cpu(voltage_object->asGpioVoltageObj.sHeader.usSize); + if (obj_size < sizeof(voltage_object->asGpioVoltageObj.sHeader) || + offset + obj_size > size) + break; if (voltage_type == voltage_object->asGpioVoltageObj.sHeader.ucVoltageType && voltage_mode == voltage_object->asGpioVoltageObj.sHeader.ucVoltageMode) return voltage_object; - offset += le16_to_cpu(voltage_object->asGpioVoltageObj.sHeader.usSize); + offset += obj_size; } return NULL; diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomfwctrl.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomfwctrl.c index 6120f14caab08e..69aee8661d1e5b 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomfwctrl.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomfwctrl.c @@ -36,16 +36,21 @@ static const union atom_voltage_object_v4 *pp_atomfwctrl_lookup_voltage_type_v4( offsetof(struct atom_voltage_objects_info_v4_1, voltage_object[0]); unsigned long start = (unsigned long)voltage_object_info_table; - while (offset < size) { + while (offset + sizeof(struct atom_voltage_object_header_v4) <= size) { const union atom_voltage_object_v4 *voltage_object = (const union atom_voltage_object_v4 *)(start + offset); + u16 obj_size; + + obj_size = le16_to_cpu(voltage_object->gpio_voltage_obj.header.object_size); + if (obj_size < sizeof(voltage_object->gpio_voltage_obj.header) || + offset + obj_size > size) + break; if (voltage_type == voltage_object->gpio_voltage_obj.header.voltage_type && voltage_mode == voltage_object->gpio_voltage_obj.header.voltage_mode) return voltage_object; - offset += le16_to_cpu(voltage_object->gpio_voltage_obj.header.object_size); - + offset += obj_size; } return NULL; diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c index f8a5648095d17a..1e9fa0a2507502 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c @@ -2216,12 +2216,24 @@ static int smu7_patch_voltage_dependency_tables_with_lookup_table( if (data->vdd_gfx_control == SMU7_VOLTAGE_CONTROL_BY_SVID2) { for (entry_id = 0; entry_id < sclk_table->count; ++entry_id) { voltage_id = sclk_table->entries[entry_id].vddInd; + if (voltage_id >= table_info->vddgfx_lookup_table->count) { + pr_err("amdgpu: sclk[%u] vddgfx index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddgfx_lookup_table->count); + return -EINVAL; + } sclk_table->entries[entry_id].vddgfx = table_info->vddgfx_lookup_table->entries[voltage_id].us_vdd; } } else { for (entry_id = 0; entry_id < sclk_table->count; ++entry_id) { voltage_id = sclk_table->entries[entry_id].vddInd; + if (voltage_id >= table_info->vddc_lookup_table->count) { + pr_err("amdgpu: sclk[%u] vddc index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddc_lookup_table->count); + return -EINVAL; + } sclk_table->entries[entry_id].vddc = table_info->vddc_lookup_table->entries[voltage_id].us_vdd; } @@ -2229,12 +2241,24 @@ static int smu7_patch_voltage_dependency_tables_with_lookup_table( for (entry_id = 0; entry_id < mclk_table->count; ++entry_id) { voltage_id = mclk_table->entries[entry_id].vddInd; + if (voltage_id >= table_info->vddc_lookup_table->count) { + pr_err("amdgpu: mclk[%u] vddc index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddc_lookup_table->count); + return -EINVAL; + } mclk_table->entries[entry_id].vddc = table_info->vddc_lookup_table->entries[voltage_id].us_vdd; } for (entry_id = 0; entry_id < mm_table->count; ++entry_id) { voltage_id = mm_table->entries[entry_id].vddcInd; + if (voltage_id >= table_info->vddc_lookup_table->count) { + pr_err("amdgpu: mm[%u] vddc index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddc_lookup_table->count); + return -EINVAL; + } mm_table->entries[entry_id].vddc = table_info->vddc_lookup_table->entries[voltage_id].us_vdd; } diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c index c283309efe87f5..ae8e44b796a895 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c @@ -685,10 +685,18 @@ static int vega10_patch_voltage_dependency_tables_with_lookup_table( case 3: vdt = table_info->vdd_dep_on_pixclk; break; case 4: vdt = table_info->vdd_dep_on_dispclk; break; case 5: vdt = table_info->vdd_dep_on_phyclk; break; + default: + continue; } for (entry_id = 0; entry_id < vdt->count; entry_id++) { voltage_id = vdt->entries[entry_id].vddInd; + if (voltage_id >= table_info->vddc_lookup_table->count) { + pr_err("amdgpu: clk_dep[%u][%u] vddc index %u out of bounds (%u)\n", + i, entry_id, voltage_id, + table_info->vddc_lookup_table->count); + return -EINVAL; + } vdt->entries[entry_id].vddc = table_info->vddc_lookup_table->entries[voltage_id].us_vdd; } @@ -696,23 +704,48 @@ static int vega10_patch_voltage_dependency_tables_with_lookup_table( for (entry_id = 0; entry_id < mm_table->count; ++entry_id) { voltage_id = mm_table->entries[entry_id].vddcInd; + if (voltage_id >= table_info->vddc_lookup_table->count) { + pr_err("amdgpu: mm[%u] vddc index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddc_lookup_table->count); + return -EINVAL; + } mm_table->entries[entry_id].vddc = table_info->vddc_lookup_table->entries[voltage_id].us_vdd; } for (entry_id = 0; entry_id < mclk_table->count; ++entry_id) { voltage_id = mclk_table->entries[entry_id].vddInd; + if (voltage_id >= table_info->vddc_lookup_table->count) { + pr_err("amdgpu: mclk[%u] vddc index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddc_lookup_table->count); + return -EINVAL; + } mclk_table->entries[entry_id].vddc = table_info->vddc_lookup_table->entries[voltage_id].us_vdd; + voltage_id = mclk_table->entries[entry_id].vddciInd; + if (voltage_id >= table_info->vddci_lookup_table->count) { + pr_err("amdgpu: mclk[%u] vddci index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddci_lookup_table->count); + return -EINVAL; + } mclk_table->entries[entry_id].vddci = table_info->vddci_lookup_table->entries[voltage_id].us_vdd; + voltage_id = mclk_table->entries[entry_id].mvddInd; + if (voltage_id >= table_info->vddmem_lookup_table->count) { + pr_err("amdgpu: mclk[%u] vddmem index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddmem_lookup_table->count); + return -EINVAL; + } mclk_table->entries[entry_id].mvdd = table_info->vddmem_lookup_table->entries[voltage_id].us_vdd; } - return 0; } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c index 118eda5429ff73..68bd673c0573d6 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c @@ -1915,7 +1915,7 @@ static int smu_v15_0_8_set_performance_level(struct smu_context *smu, struct smu_dpm_table *gfx_table = &dpm_context->dpm_tables.gfx_table; struct smu_dpm_table *uclk_table = &dpm_context->dpm_tables.uclk_table; struct smu_umd_pstate_table *pstate_table = &smu->pstate_table; - int ret; + int ret = 0; switch (level) { case AMD_DPM_FORCED_LEVEL_PERF_DETERMINISM: @@ -1956,9 +1956,6 @@ static int smu_v15_0_8_set_performance_level(struct smu_context *smu, SMU_DPM_TABLE_MAX(uclk_table); } - if (ret) - goto out; - smu_cmn_reset_custom_level(smu); break; diff --git a/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c b/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c index cf90d4468b5c1e..344c3f4660185a 100644 --- a/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c +++ b/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c @@ -1015,7 +1015,7 @@ cdns_dsi_bridge_atomic_reset(struct drm_bridge *bridge) dsi_state = kzalloc_obj(*dsi_state); if (!dsi_state) - return NULL; + return ERR_PTR(-ENOMEM); memset(dsi_state, 0, sizeof(*dsi_state)); dsi_state->base.bridge = bridge; diff --git a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c index 36c07b71fe04bf..46779b49545bdb 100644 --- a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c +++ b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c @@ -1927,7 +1927,7 @@ cdns_mhdp_bridge_atomic_reset(struct drm_bridge *bridge) cdns_mhdp_state = kzalloc_obj(*cdns_mhdp_state); if (!cdns_mhdp_state) - return NULL; + return ERR_PTR(-ENOMEM); __drm_atomic_helper_bridge_reset(bridge, &cdns_mhdp_state->base); diff --git a/drivers/gpu/drm/bridge/display-connector.c b/drivers/gpu/drm/bridge/display-connector.c index 6b128fabe3a97b..441355ace2bb89 100644 --- a/drivers/gpu/drm/bridge/display-connector.c +++ b/drivers/gpu/drm/bridge/display-connector.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -25,6 +26,8 @@ struct display_connector { struct regulator *supply; struct gpio_desc *ddc_en; + + struct work_struct hpd_work; }; static inline struct display_connector * @@ -94,6 +97,34 @@ display_connector_bridge_detect(struct drm_bridge *bridge, struct drm_connector return display_connector_detect(bridge); } +static void display_connector_hpd_enable(struct drm_bridge *bridge) +{ + struct display_connector *conn = to_display_connector(bridge); + + enable_irq(conn->hpd_irq); + + if (conn->bridge.type == DRM_MODE_CONNECTOR_DisplayPort) + schedule_work(&conn->hpd_work); +} + +static void display_connector_hpd_disable(struct drm_bridge *bridge) +{ + struct display_connector *conn = to_display_connector(bridge); + + if (conn->bridge.type == DRM_MODE_CONNECTOR_DisplayPort) + cancel_work_sync(&conn->hpd_work); + + disable_irq(conn->hpd_irq); +} + +static void display_connector_hpd_work(struct work_struct *work) +{ + struct display_connector *conn = container_of(work, struct display_connector, hpd_work); + struct drm_bridge *bridge = &conn->bridge; + + drm_bridge_hpd_notify(bridge, display_connector_detect(bridge)); +} + static const struct drm_edid *display_connector_edid_read(struct drm_bridge *bridge, struct drm_connector *connector) { @@ -186,6 +217,8 @@ static const struct drm_bridge_funcs display_connector_bridge_funcs = { .attach = display_connector_attach, .destroy = display_connector_destroy, .detect = display_connector_bridge_detect, + .hpd_enable = display_connector_hpd_enable, + .hpd_disable = display_connector_hpd_disable, .edid_read = display_connector_edid_read, .atomic_get_output_bus_fmts = display_connector_get_output_bus_fmts, .atomic_get_input_bus_fmts = display_connector_get_input_bus_fmts, @@ -315,6 +348,7 @@ static int display_connector_probe(struct platform_device *pdev) NULL, display_connector_hpd_irq, IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING | + IRQF_NO_AUTOEN | IRQF_ONESHOT, "HPD", conn); if (ret) { @@ -386,6 +420,8 @@ static int display_connector_probe(struct platform_device *pdev) conn->bridge.ops |= DRM_BRIDGE_OP_DETECT; if (conn->hpd_irq >= 0) conn->bridge.ops |= DRM_BRIDGE_OP_HPD; + if (conn->hpd_irq >= 0 && type == DRM_MODE_CONNECTOR_DisplayPort) + INIT_WORK(&conn->hpd_work, display_connector_hpd_work); dev_dbg(&pdev->dev, "Found %s display connector '%s' %s DDC bus and %s HPD GPIO (ops 0x%x)\n", diff --git a/drivers/gpu/drm/bridge/of-display-mode-bridge.c b/drivers/gpu/drm/bridge/of-display-mode-bridge.c index cb15713f3a7965..e66dae168fd0df 100644 --- a/drivers/gpu/drm/bridge/of-display-mode-bridge.c +++ b/drivers/gpu/drm/bridge/of-display-mode-bridge.c @@ -51,7 +51,7 @@ static int of_display_mode_bridge_get_modes(struct drm_bridge *bridge, return 0; } -struct drm_bridge_funcs of_display_mode_bridge_funcs = { +static const struct drm_bridge_funcs of_display_mode_bridge_funcs = { .attach = of_display_mode_bridge_attach, .get_modes = of_display_mode_bridge_get_modes, }; diff --git a/drivers/gpu/drm/bridge/synopsys/dw-dp.c b/drivers/gpu/drm/bridge/synopsys/dw-dp.c index 21541be094c47e..36ee6e027af528 100644 --- a/drivers/gpu/drm/bridge/synopsys/dw-dp.c +++ b/drivers/gpu/drm/bridge/synopsys/dw-dp.c @@ -2093,6 +2093,12 @@ unregister_aux: } EXPORT_SYMBOL_GPL(dw_dp_bind); +void dw_dp_unbind(struct dw_dp *dp) +{ + drm_dp_aux_unregister(&dp->aux); +} +EXPORT_SYMBOL_GPL(dw_dp_unbind); + MODULE_AUTHOR("Andy Yan "); MODULE_DESCRIPTION("DW DP Core Library"); MODULE_LICENSE("GPL"); diff --git a/drivers/gpu/drm/bridge/tc358767.c b/drivers/gpu/drm/bridge/tc358767.c index 7188935fdb8261..948bb7b2867a0a 100644 --- a/drivers/gpu/drm/bridge/tc358767.c +++ b/drivers/gpu/drm/bridge/tc358767.c @@ -527,7 +527,7 @@ static ssize_t tc_aux_transfer(struct drm_dp_aux *aux, * address-only transfer */ if (size) - size = FIELD_GET(AUX_BYTES, auxstatus); + size = min_t(size_t, size, FIELD_GET(AUX_BYTES, auxstatus)); msg->reply = FIELD_GET(AUX_STATUS, auxstatus); switch (request) { diff --git a/drivers/gpu/drm/display/drm_hdmi_state_helper.c b/drivers/gpu/drm/display/drm_hdmi_state_helper.c index cae0d85fb44079..a3aee41385548d 100644 --- a/drivers/gpu/drm/display/drm_hdmi_state_helper.c +++ b/drivers/gpu/drm/display/drm_hdmi_state_helper.c @@ -910,8 +910,21 @@ drm_hdmi_connector_mode_valid(struct drm_connector *connector, const struct drm_display_mode *mode) { unsigned long long clock; + enum drm_output_color_format fmt; + + if (drm_mode_is_420_only(&connector->display_info, mode)) { + if (connector->ycbcr_420_allowed) + fmt = DRM_OUTPUT_COLOR_FORMAT_YCBCR420; + else + return MODE_NO_420; + } else if (drm_mode_is_420_also(&connector->display_info, mode) && + connector->ycbcr_420_allowed) { + fmt = DRM_OUTPUT_COLOR_FORMAT_YCBCR420; + } else { + fmt = DRM_OUTPUT_COLOR_FORMAT_RGB444; + } - clock = drm_hdmi_compute_mode_clock(mode, 8, DRM_OUTPUT_COLOR_FORMAT_RGB444); + clock = drm_hdmi_compute_mode_clock(mode, 8, fmt); if (!clock) return MODE_ERROR; diff --git a/drivers/gpu/drm/drm_atomic_state_helper.c b/drivers/gpu/drm/drm_atomic_state_helper.c index cc70508d4fdba1..a82568d87e4f76 100644 --- a/drivers/gpu/drm/drm_atomic_state_helper.c +++ b/drivers/gpu/drm/drm_atomic_state_helper.c @@ -731,8 +731,6 @@ void __drm_atomic_helper_private_obj_create_state(struct drm_private_obj *obj, { if (state) state->obj = obj; - - obj->state = state; } EXPORT_SYMBOL(__drm_atomic_helper_private_obj_create_state); diff --git a/drivers/gpu/drm/drm_dumb_buffers.c b/drivers/gpu/drm/drm_dumb_buffers.c index 2156dbe601c9c4..8e9ff17538e731 100644 --- a/drivers/gpu/drm/drm_dumb_buffers.c +++ b/drivers/gpu/drm/drm_dumb_buffers.c @@ -32,7 +32,6 @@ #include #include "drm_crtc_internal.h" -#include "drm_internal.h" /** * DOC: overview diff --git a/drivers/gpu/drm/lima/lima_device.c b/drivers/gpu/drm/lima/lima_device.c index 0bf7105c8748b4..7c873e62c16da4 100644 --- a/drivers/gpu/drm/lima/lima_device.c +++ b/drivers/gpu/drm/lima/lima_device.c @@ -368,12 +368,6 @@ int lima_device_init(struct lima_device *ldev) if (err) goto err_out0; - ldev->empty_vm = lima_vm_create(ldev); - if (!ldev->empty_vm) { - err = -ENOMEM; - goto err_out1; - } - ldev->va_start = 0; if (ldev->id == lima_gpu_mali450) { ldev->va_end = LIMA_VA_RESERVE_START; @@ -387,6 +381,12 @@ int lima_device_init(struct lima_device *ldev) } else ldev->va_end = LIMA_VA_RESERVE_END; + ldev->empty_vm = lima_vm_create(ldev); + if (!ldev->empty_vm) { + err = -ENOMEM; + goto err_out1; + } + ldev->iomem = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(ldev->iomem)) { dev_err(ldev->dev, "fail to ioremap iomem\n"); diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c index 2c0bbac43c52b1..f1df2514c6132b 100644 --- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c @@ -1275,6 +1275,11 @@ static irqreturn_t a5xx_irq(struct msm_gpu *gpu) status & ~A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR); if (priv->disable_err_irq) { + /* Turn off interrupts to avoid interrupt storm */ + gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, + A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | + A5XX_RBBM_INT_0_MASK_CP_SW); + status &= A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | A5XX_RBBM_INT_0_MASK_CP_SW; } diff --git a/drivers/gpu/drm/msm/adreno/a6xx_catalog.c b/drivers/gpu/drm/msm/adreno/a6xx_catalog.c index 3e6f409d13a2a5..a98d550b72d0e5 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_catalog.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_catalog.c @@ -1199,7 +1199,7 @@ static const struct adreno_reglist a730_hwcg[] = { { REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080 }, { REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222220 }, { REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222 }, - { REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222 }, + { REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22220222 }, { REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00222222 }, { REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777 }, { REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777 }, diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c index 2e5d7b53a0c380..a2f6918c4f7f23 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c @@ -642,7 +642,7 @@ static void a6xx_rpmh_stop(struct a6xx_gmu *gmu) int ret; u32 val; - if (test_and_clear_bit(GMU_STATUS_FW_START, &gmu->status)) + if (!test_and_clear_bit(GMU_STATUS_FW_START, &gmu->status)) return; if (adreno_is_a840(adreno_gpu)) @@ -1465,6 +1465,9 @@ static void a6xx_gmu_shutdown(struct a6xx_gmu *gmu) /* Stop the interrupts and mask the hardware */ a6xx_gmu_irq_disable(gmu); + /* Halt the gmu cm3 core */ + gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1); + /* Tell RPMh to power off the GPU */ a6xx_rpmh_stop(gmu); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c index 8b3bb2fd433bab..e293b4ca808a46 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c @@ -1911,8 +1911,11 @@ static irqreturn_t a6xx_irq(struct msm_gpu *gpu) gpu_write(gpu, REG_A6XX_RBBM_INT_CLEAR_CMD, status); - if (priv->disable_err_irq) + if (priv->disable_err_irq) { + /* Turn off interrupts to avoid interrupt storm */ + gpu_write(gpu, REG_A6XX_RBBM_INT_0_MASK, A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS); status &= A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS; + } if (status & A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT) a6xx_fault_detect_irq(gpu); @@ -2770,8 +2773,9 @@ static struct msm_gpu *a6xx_gpu_init(struct drm_device *dev) adreno_gpu->ubwc_config = qcom_ubwc_config_get_data(); if (IS_ERR(adreno_gpu->ubwc_config)) { + ret = PTR_ERR(adreno_gpu->ubwc_config); a6xx_destroy(&(a6xx_gpu->base.base)); - return ERR_CAST(adreno_gpu->ubwc_config); + return ERR_PTR(ret); } /* Set up the preemption specific bits and pieces for each ringbuffer */ diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c index 166365359fa6dd..3ea8ff8c740449 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c @@ -1244,9 +1244,12 @@ static void a6xx_get_gmu_registers(struct msm_gpu *gpu, _a6xx_get_gmu_registers(gpu, a6xx_state, &a6xx_gmu_reglist[1], &a6xx_state->gmu_registers[1], true); - if (adreno_is_a621(adreno_gpu) || adreno_is_a623(adreno_gpu)) + if (adreno_is_a621(adreno_gpu)) _a6xx_get_gmu_registers(gpu, a6xx_state, &a621_gpucc_reg, &a6xx_state->gmu_registers[2], false); + else if (adreno_is_a623(adreno_gpu) || adreno_is_a663(adreno_gpu)) + _a6xx_get_gmu_registers(gpu, a6xx_state, &a623_gpucc_reg, + &a6xx_state->gmu_registers[2], false); else _a6xx_get_gmu_registers(gpu, a6xx_state, &a6xx_gpucc_reg, &a6xx_state->gmu_registers[2], false); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h index b49d8427b59e6d..0a13a65f89ac8d 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h @@ -377,6 +377,17 @@ static const u32 a6xx_gmu_gpucc_registers[] = { }; static const u32 a621_gmu_gpucc_registers[] = { + /* GPU CC */ + 0x24000, 0x2400e, 0x24400, 0x2440e, 0x24800, 0x24805, 0x24c00, 0x24cff, + 0x25800, 0x25804, 0x25c00, 0x25c04, 0x26000, 0x26004, 0x26400, 0x26405, + 0x26414, 0x2641d, 0x2642a, 0x26430, 0x26432, 0x26432, 0x26441, 0x26455, + 0x26466, 0x26468, 0x26478, 0x2647a, 0x26489, 0x2648a, 0x2649c, 0x2649e, + 0x264a0, 0x264a3, 0x264b3, 0x264b5, 0x264c5, 0x264c7, 0x264d6, 0x264d8, + 0x264e8, 0x264e9, 0x264f9, 0x264fc, 0x2650b, 0x2650c, 0x2651c, 0x2651e, + 0x26540, 0x26570, 0x26600, 0x26616, 0x26620, 0x2662d, +}; + +static const u32 a623_gmu_gpucc_registers[] = { /* GPU CC */ 0x24000, 0x2400e, 0x24400, 0x2440e, 0x25800, 0x25804, 0x25c00, 0x25c04, 0x26000, 0x26004, 0x26400, 0x26405, 0x26414, 0x2641d, 0x2642a, 0x26430, @@ -402,6 +413,7 @@ static const struct a6xx_registers a6xx_gmu_reglist[] = { static const struct a6xx_registers a6xx_gpucc_reg = REGS(a6xx_gmu_gpucc_registers, 0, 0); static const struct a6xx_registers a621_gpucc_reg = REGS(a621_gmu_gpucc_registers, 0, 0); +static const struct a6xx_registers a623_gpucc_reg = REGS(a623_gmu_gpucc_registers, 0, 0); static u32 a6xx_get_cp_roq_size(struct msm_gpu *gpu); static u32 a7xx_get_cp_roq_size(struct msm_gpu *gpu); diff --git a/drivers/gpu/drm/msm/adreno/a8xx_gpu.c b/drivers/gpu/drm/msm/adreno/a8xx_gpu.c index 9e44fd1ae63463..0f6fd35bd5878b 100644 --- a/drivers/gpu/drm/msm/adreno/a8xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a8xx_gpu.c @@ -1211,8 +1211,11 @@ irqreturn_t a8xx_irq(struct msm_gpu *gpu) gpu_write(gpu, REG_A8XX_RBBM_INT_CLEAR_CMD, status); - if (priv->disable_err_irq) + if (priv->disable_err_irq) { + /* Turn off interrupts to avoid interrupt storm */ + gpu_write(gpu, REG_A8XX_RBBM_INT_0_MASK, A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS); status &= A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS; + } if (status & A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT) a8xx_fault_detect_irq(gpu); diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h index 06da1583fb1eb4..85f455a9f29c8d 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h @@ -86,42 +86,42 @@ static const struct dpu_sspp_cfg kaanapali_sspp[] = { .name = "sspp_8", .id = SSPP_DMA0, .base = 0x97000, .len = 0x84, .features = DMA_SDM845_MASK_SDMA, - .sblk = &dpu_dma_sblk, + .sblk = &dpu_dma_sblk_v13, .xin_id = 1, .type = SSPP_TYPE_DMA, }, { .name = "sspp_9", .id = SSPP_DMA1, .base = 0xa0000, .len = 0x84, .features = DMA_SDM845_MASK_SDMA, - .sblk = &dpu_dma_sblk, + .sblk = &dpu_dma_sblk_v13, .xin_id = 5, .type = SSPP_TYPE_DMA, }, { .name = "sspp_10", .id = SSPP_DMA2, .base = 0xa9000, .len = 0x84, .features = DMA_SDM845_MASK_SDMA, - .sblk = &dpu_dma_sblk, + .sblk = &dpu_dma_sblk_v13, .xin_id = 9, .type = SSPP_TYPE_DMA, }, { .name = "sspp_11", .id = SSPP_DMA3, .base = 0xb2000, .len = 0x84, .features = DMA_SDM845_MASK_SDMA, - .sblk = &dpu_dma_sblk, + .sblk = &dpu_dma_sblk_v13, .xin_id = 13, .type = SSPP_TYPE_DMA, }, { .name = "sspp_12", .id = SSPP_DMA4, .base = 0xbb000, .len = 0x84, .features = DMA_CURSOR_SDM845_MASK_SDMA, - .sblk = &dpu_dma_sblk, + .sblk = &dpu_dma_sblk_v13, .xin_id = 14, .type = SSPP_TYPE_DMA, }, { .name = "sspp_13", .id = SSPP_DMA5, .base = 0xc4000, .len = 0x84, .features = DMA_CURSOR_SDM845_MASK_SDMA, - .sblk = &dpu_dma_sblk, + .sblk = &dpu_dma_sblk_v13, .xin_id = 15, .type = SSPP_TYPE_DMA, }, diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c index 13cc658065c56a..6524531bd8bdc3 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c @@ -394,6 +394,10 @@ int dpu_core_perf_crtc_update(struct drm_crtc *crtc, trace_dpu_core_perf_update_clk(kms->dev, !crtc->enabled, clk_rate); + /* If we're going offline, PM callbacks will disable the clocks instead */ + if (!clk_rate) + return 0; + clk_rate = min(clk_rate, kms->perf.max_core_clk_rate); ret = dev_pm_opp_set_rate(&kms->pdev->dev, clk_rate); if (ret) { diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c index 778e231d496777..1f20695f81e35c 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c @@ -710,8 +710,7 @@ void dpu_encoder_update_topology(struct drm_encoder *drm_enc, if (fb && MSM_FORMAT_IS_YUV(msm_framebuffer_format(fb))) topology->num_cdm++; } else if (disp_info->intf_type == INTF_DP) { - if (msm_dp_is_yuv_420_enabled(priv->kms->dp[disp_info->h_tile_instance[0]], - adj_mode)) + if (drm_mode_is_420_only(&connector->display_info, adj_mode)) topology->num_cdm++; } } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c index 2e10add84fd71f..9a993cdfab8563 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c @@ -303,6 +303,16 @@ static const u32 wb2_formats_rgb_yuv[] = { .num_formats = ARRAY_SIZE(plane_formats), \ } +#define _DMA_SBLK_V13() \ + { \ + .sspp_rec0_blk = {.name = "sspp_rec0", \ + .base = 0x1000, .len = 0x180,}, \ + .sspp_rec1_blk = {.name = "sspp_rec1", \ + .base = 0x3000, .len = 0x180,}, \ + .format_list = plane_formats, \ + .num_formats = ARRAY_SIZE(plane_formats), \ + } + static const struct dpu_rotation_cfg dpu_rot_sc7280_cfg_v2 = { .rot_maxheight = 1088, .rot_num_formats = ARRAY_SIZE(rotation_v2_formats), @@ -353,6 +363,8 @@ static const struct dpu_sspp_sub_blks dpu_rgb_sblk = _RGB_SBLK(); static const struct dpu_sspp_sub_blks dpu_dma_sblk = _DMA_SBLK(); +static const struct dpu_sspp_sub_blks dpu_dma_sblk_v13 = _DMA_SBLK_V13(); + /************************************************************* * MIXER sub blocks config *************************************************************/ diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c index 7726edb0d4ede2..6ae49f94fea7b3 100644 --- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c +++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c @@ -398,7 +398,7 @@ static int mdp4_kms_init(struct drm_device *dev) ret = mdp_kms_init(&mdp4_kms->base, &kms_funcs); if (ret) { DRM_DEV_ERROR(dev->dev, "failed to init kms\n"); - goto fail; + return ret; } kms = priv->kms; @@ -409,7 +409,7 @@ static int mdp4_kms_init(struct drm_device *dev) ret = regulator_enable(mdp4_kms->vdd); if (ret) { DRM_DEV_ERROR(dev->dev, "failed to enable regulator vdd: %d\n", ret); - goto fail; + return ret; } } @@ -421,7 +421,7 @@ static int mdp4_kms_init(struct drm_device *dev) DRM_DEV_ERROR(dev->dev, "unexpected MDP version: v%d.%d\n", major, minor); ret = -ENXIO; - goto fail; + return ret; } mdp4_kms->rev = minor; @@ -430,7 +430,7 @@ static int mdp4_kms_init(struct drm_device *dev) if (!mdp4_kms->lut_clk) { DRM_DEV_ERROR(dev->dev, "failed to get lut_clk\n"); ret = -ENODEV; - goto fail; + return ret; } clk_set_rate(mdp4_kms->lut_clk, max_clk); } @@ -452,7 +452,7 @@ static int mdp4_kms_init(struct drm_device *dev) vm = msm_kms_init_vm(mdp4_kms->dev, NULL); if (IS_ERR(vm)) { ret = PTR_ERR(vm); - goto fail; + return ret; } kms->vm = vm; @@ -460,7 +460,7 @@ static int mdp4_kms_init(struct drm_device *dev) ret = modeset_init(mdp4_kms); if (ret) { DRM_DEV_ERROR(dev->dev, "modeset_init failed: %d\n", ret); - goto fail; + return ret; } mdp4_kms->blank_cursor_bo = msm_gem_new(dev, SZ_16K, MSM_BO_WC | MSM_BO_SCANOUT); @@ -468,14 +468,14 @@ static int mdp4_kms_init(struct drm_device *dev) ret = PTR_ERR(mdp4_kms->blank_cursor_bo); DRM_DEV_ERROR(dev->dev, "could not allocate blank-cursor bo: %d\n", ret); mdp4_kms->blank_cursor_bo = NULL; - goto fail; + return ret; } ret = msm_gem_get_and_pin_iova(mdp4_kms->blank_cursor_bo, kms->vm, &mdp4_kms->blank_cursor_iova); if (ret) { DRM_DEV_ERROR(dev->dev, "could not pin blank-cursor bo: %d\n", ret); - goto fail; + return ret; } dev->mode_config.min_width = 0; @@ -484,12 +484,6 @@ static int mdp4_kms_init(struct drm_device *dev) dev->mode_config.max_height = 2048; return 0; - -fail: - if (kms) - mdp4_destroy(kms); - - return ret; } static const struct dev_pm_ops mdp4_pm_ops = { diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c index 0a004ab9fc856c..3934cd060b2766 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c @@ -517,7 +517,7 @@ static int mdp5_kms_init(struct drm_device *dev) ret = mdp_kms_init(&mdp5_kms->base, &kms_funcs); if (ret) { DRM_DEV_ERROR(&pdev->dev, "failed to init kms\n"); - goto fail; + return ret; } config = mdp5_cfg_get_config(mdp5_kms->cfg); @@ -540,7 +540,7 @@ static int mdp5_kms_init(struct drm_device *dev) vm = msm_kms_init_vm(mdp5_kms->dev, pdev->dev.parent); if (IS_ERR(vm)) { ret = PTR_ERR(vm); - goto fail; + return ret; } kms->vm = vm; @@ -550,7 +550,7 @@ static int mdp5_kms_init(struct drm_device *dev) ret = modeset_init(mdp5_kms); if (ret) { DRM_DEV_ERROR(&pdev->dev, "modeset_init failed: %d\n", ret); - goto fail; + return ret; } dev->mode_config.min_width = 0; @@ -562,11 +562,6 @@ static int mdp5_kms_init(struct drm_device *dev) dev->vblank_disable_immediate = true; return 0; -fail: - if (kms) - mdp5_kms_destroy(kms); - - return ret; } static void mdp5_destroy(struct mdp5_kms *mdp5_kms) diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.c b/drivers/gpu/drm/msm/dp/dp_ctrl.c index 86ef8c89ad4437..9024abdc2c569f 100644 --- a/drivers/gpu/drm/msm/dp/dp_ctrl.c +++ b/drivers/gpu/drm/msm/dp/dp_ctrl.c @@ -1950,13 +1950,12 @@ static int msm_dp_ctrl_reinitialize_mainlink(struct msm_dp_ctrl_private *ctrl) msm_dp_ctrl_mainlink_disable(ctrl); ctrl->phy_opts.dp.lanes = ctrl->link->link_params.num_lanes; phy_configure(phy, &ctrl->phy_opts); + /* * Disable and re-enable the mainlink clock since the * link clock might have been adjusted as part of the * link maintenance. */ - dev_pm_opp_set_rate(ctrl->dev, 0); - msm_dp_ctrl_link_clk_disable(&ctrl->msm_dp_ctrl); phy_power_off(phy); @@ -1982,7 +1981,6 @@ static int msm_dp_ctrl_deinitialize_mainlink(struct msm_dp_ctrl_private *ctrl) msm_dp_ctrl_reset(&ctrl->msm_dp_ctrl); - dev_pm_opp_set_rate(ctrl->dev, 0); msm_dp_ctrl_link_clk_disable(&ctrl->msm_dp_ctrl); phy_power_off(phy); @@ -2602,7 +2600,6 @@ void msm_dp_ctrl_off(struct msm_dp_ctrl *msm_dp_ctrl) ctrl->stream_clks_on = false; } - dev_pm_opp_set_rate(ctrl->dev, 0); msm_dp_ctrl_link_clk_disable(&ctrl->msm_dp_ctrl); phy_power_off(phy); diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c index dc6f33809ca5f0..79e2b171e269a3 100644 --- a/drivers/gpu/drm/msm/dp/dp_display.c +++ b/drivers/gpu/drm/msm/dp/dp_display.c @@ -698,6 +698,7 @@ enum drm_mode_status msm_dp_bridge_mode_valid(struct drm_bridge *bridge, u32 mode_rate_khz = 0, supported_rate_khz = 0, mode_bpp = 0; struct msm_dp *dp; int mode_pclk_khz = mode->clock; + bool is_yuv_420; dp = to_dp_bridge(bridge)->msm_dp_display; @@ -709,9 +710,16 @@ enum drm_mode_status msm_dp_bridge_mode_valid(struct drm_bridge *bridge, msm_dp_display = container_of(dp, struct msm_dp_display_private, msm_dp_display); link_info = &msm_dp_display->panel->link_info; - if ((drm_mode_is_420_only(&dp->connector->display_info, mode) && - msm_dp_display->panel->vsc_sdp_supported) || - msm_dp_wide_bus_available(dp)) + is_yuv_420 = drm_mode_is_420_only(&dp->connector->display_info, mode); + + /* + * YUV 420 is carried over DP by signalling the colorimetry through a + * VSC SDP, so a 420-only mode cannot be driven without VSC SDP support. + */ + if (is_yuv_420 && !msm_dp_display->panel->vsc_sdp_supported) + return MODE_NO_420; + + if (is_yuv_420 || msm_dp_display->wide_bus_supported) mode_pclk_khz /= 2; if (mode_pclk_khz > DP_MAX_PIXEL_CLK_KHZ) @@ -1277,22 +1285,10 @@ void __exit msm_dp_unregister(void) platform_driver_unregister(&msm_dp_display_driver); } -bool msm_dp_is_yuv_420_enabled(const struct msm_dp *msm_dp_display, - const struct drm_display_mode *mode) -{ - struct msm_dp_display_private *dp; - const struct drm_display_info *info; - - dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); - info = &msm_dp_display->connector->display_info; - - return dp->panel->vsc_sdp_supported && drm_mode_is_420_only(info, mode); -} - bool msm_dp_needs_periph_flush(const struct msm_dp *msm_dp_display, const struct drm_display_mode *mode) { - return msm_dp_is_yuv_420_enabled(msm_dp_display, mode); + return drm_mode_is_420_only(&msm_dp_display->connector->display_info, mode); } bool msm_dp_wide_bus_available(const struct msm_dp *msm_dp_display) diff --git a/drivers/gpu/drm/msm/dp/dp_panel.c b/drivers/gpu/drm/msm/dp/dp_panel.c index 6bb021820d7c53..5b4954e7cb7480 100644 --- a/drivers/gpu/drm/msm/dp/dp_panel.c +++ b/drivers/gpu/drm/msm/dp/dp_panel.c @@ -332,8 +332,10 @@ int msm_dp_panel_get_modes(struct msm_dp_panel *msm_dp_panel, return -EINVAL; } - if (msm_dp_panel->drm_edid) + if (msm_dp_panel->drm_edid) { + drm_edid_connector_update(connector, msm_dp_panel->drm_edid); return drm_edid_connector_add_modes(connector); + } return 0; } diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c index 5e1b313f04c0db..2685cc3d035905 100644 --- a/drivers/gpu/drm/msm/dsi/dsi_host.c +++ b/drivers/gpu/drm/msm/dsi/dsi_host.c @@ -549,8 +549,6 @@ error: void dsi_link_clk_disable_6g(struct msm_dsi_host *msm_host) { - /* Drop the performance state vote */ - dev_pm_opp_set_rate(&msm_host->pdev->dev, 0); clk_disable_unprepare(msm_host->esc_clk); clk_disable_unprepare(msm_host->pixel_clk); clk_disable_unprepare(msm_host->byte_intf_clk); diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h index 21a59d66e8dc55..f5d3e806f8fd51 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h @@ -111,6 +111,7 @@ struct msm_dsi_phy { struct msm_dsi_dphy_timing timing; const struct msm_dsi_phy_cfg *cfg; void *tuning_cfg; + void *pll_data; enum msm_dsi_phy_usecase usecase; bool regulator_ldo_mode; diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c index 984a66085dfbf8..5d805a797abdb1 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c @@ -426,8 +426,11 @@ static void dsi_pll_enable_pll_bias(struct dsi_pll_7nm *pll) u32 data; spin_lock_irqsave(&pll->pll_enable_lock, flags); - pll->pll_enable_cnt++; - WARN_ON(pll->pll_enable_cnt == INT_MAX); + if (pll->pll_enable_cnt++) { + spin_unlock_irqrestore(&pll->pll_enable_lock, flags); + WARN_ON(pll->pll_enable_cnt == INT_MAX); + return; + } data = readl(pll->phy->base + REG_DSI_7nm_PHY_CMN_CTRL_0); data |= DSI_7nm_PHY_CMN_CTRL_0_PLL_SHUTDOWNB; @@ -873,6 +876,7 @@ static int dsi_pll_7nm_init(struct msm_dsi_phy *phy) spin_lock_init(&pll_7nm->pll_enable_lock); pll_7nm->phy = phy; + phy->pll_data = pll_7nm; ret = pll_7nm_register(pll_7nm, phy->provided_clocks->hws); if (ret) { @@ -961,8 +965,10 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy, u32 const delay_us = 5; u32 const timeout_us = 1000; struct msm_dsi_dphy_timing *timing = &phy->timing; + struct dsi_pll_7nm *pll = phy->pll_data; void __iomem *base = phy->base; bool less_than_1500_mhz; + unsigned long flags; u32 vreg_ctrl_0, vreg_ctrl_1, lane_ctrl0; u32 glbl_pemph_ctrl_0; u32 glbl_str_swi_cal_sel_ctrl, glbl_hstx_str_ctrl_0; @@ -1084,10 +1090,13 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy, glbl_rescode_bot_ctrl = 0x3c; } + spin_lock_irqsave(&pll->pll_enable_lock, flags); + pll->pll_enable_cnt = 1; /* de-assert digital and pll power down */ data = DSI_7nm_PHY_CMN_CTRL_0_DIGTOP_PWRDN_B | DSI_7nm_PHY_CMN_CTRL_0_PLL_SHUTDOWNB; writel(data, base + REG_DSI_7nm_PHY_CMN_CTRL_0); + spin_unlock_irqrestore(&pll->pll_enable_lock, flags); /* Assert PLL core reset */ writel(0x00, base + REG_DSI_7nm_PHY_CMN_PLL_CNTRL); @@ -1200,7 +1209,9 @@ static bool dsi_7nm_set_continuous_clock(struct msm_dsi_phy *phy, bool enable) static void dsi_7nm_phy_disable(struct msm_dsi_phy *phy) { + struct dsi_pll_7nm *pll = phy->pll_data; void __iomem *base = phy->base; + unsigned long flags; u32 data; DBG(""); @@ -1227,8 +1238,11 @@ static void dsi_7nm_phy_disable(struct msm_dsi_phy *phy) writel(data, base + REG_DSI_7nm_PHY_CMN_CTRL_0); writel(0, base + REG_DSI_7nm_PHY_CMN_LANE_CTRL0); + spin_lock_irqsave(&pll->pll_enable_lock, flags); + pll->pll_enable_cnt = 0; /* Turn off all PHY blocks */ writel(0x00, base + REG_DSI_7nm_PHY_CMN_CTRL_0); + spin_unlock_irqrestore(&pll->pll_enable_lock, flags); /* make sure phy is turned off */ wmb(); diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h index 3787db8770adae..3d5679be488e75 100644 --- a/drivers/gpu/drm/msm/msm_drv.h +++ b/drivers/gpu/drm/msm/msm_drv.h @@ -356,8 +356,6 @@ void __exit msm_dp_unregister(void); int msm_dp_modeset_init(struct msm_dp *dp_display, struct drm_device *dev, struct drm_encoder *encoder, bool yuv_supported); void msm_dp_snapshot(struct msm_disp_state *disp_state, struct msm_dp *dp_display); -bool msm_dp_is_yuv_420_enabled(const struct msm_dp *dp_display, - const struct drm_display_mode *mode); bool msm_dp_needs_periph_flush(const struct msm_dp *dp_display, const struct drm_display_mode *mode); bool msm_dp_wide_bus_available(const struct msm_dp *dp_display); @@ -382,12 +380,6 @@ static inline void msm_dp_snapshot(struct msm_disp_state *disp_state, struct msm { } -static inline bool msm_dp_is_yuv_420_enabled(const struct msm_dp *dp_display, - const struct drm_display_mode *mode) -{ - return false; -} - static inline bool msm_dp_needs_periph_flush(const struct msm_dp *dp_display, const struct drm_display_mode *mode) { diff --git a/drivers/gpu/drm/msm/msm_gem_vma.c b/drivers/gpu/drm/msm/msm_gem_vma.c index c4cfe036066b75..7441d4a01d31d4 100644 --- a/drivers/gpu/drm/msm/msm_gem_vma.c +++ b/drivers/gpu/drm/msm/msm_gem_vma.c @@ -458,6 +458,8 @@ msm_gem_vm_bo_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec) return ret; } + drm_gpuvm_bo_evict(vm_bo, false); + return 0; } diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c index 18ed00e5f143b9..03c05785606586 100644 --- a/drivers/gpu/drm/msm/msm_gpu.c +++ b/drivers/gpu/drm/msm/msm_gpu.c @@ -505,6 +505,8 @@ static void recover_worker(struct kthread_work *work) */ if (!vm->managed) msm_gem_vm_unusable(submit->vm); + + put_task_struct(task); } noreclaim_flag = memalloc_noreclaim_save(); @@ -552,11 +554,13 @@ static void recover_worker(struct kthread_work *work) msm_update_fence(ring->fctx, fence); } - /* retire completed submits, plus the one that hung: */ - retire_submits(gpu); + priv->disable_err_irq = false; gpu->funcs->recover(gpu); + /* retire completed submits, plus the one that hung: */ + retire_submits(gpu); + /* * Replay all remaining submits starting with highest priority * ring diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.c b/drivers/gpu/drm/msm/msm_ringbuffer.c index 2d6b930b766ecf..59c69aa75649e8 100644 --- a/drivers/gpu/drm/msm/msm_ringbuffer.c +++ b/drivers/gpu/drm/msm/msm_ringbuffer.c @@ -109,9 +109,9 @@ struct msm_ringbuffer *msm_ringbuffer_new(struct msm_gpu *gpu, int id, ring->memptrs_iova = memptrs_iova; ret = drm_sched_init(&ring->sched, &args); - if (ret) { + if (ret) goto fail; - } + ring->sched_initialized = true; INIT_LIST_HEAD(&ring->submits); spin_lock_init(&ring->submit_lock); @@ -133,7 +133,8 @@ void msm_ringbuffer_destroy(struct msm_ringbuffer *ring) if (IS_ERR_OR_NULL(ring)) return; - drm_sched_fini(&ring->sched); + if (ring->sched_initialized) + drm_sched_fini(&ring->sched); msm_fence_context_free(ring->fctx); diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.h b/drivers/gpu/drm/msm/msm_ringbuffer.h index 28ca8c9f7463de..3631ec283c6e55 100644 --- a/drivers/gpu/drm/msm/msm_ringbuffer.h +++ b/drivers/gpu/drm/msm/msm_ringbuffer.h @@ -56,6 +56,7 @@ struct msm_ringbuffer { * The job scheduler for this ring. */ struct drm_gpu_scheduler sched; + bool sched_initialized; /* * List of in-flight submits on this ring. Protected by submit_lock. diff --git a/drivers/gpu/drm/mxsfb/lcdif_kms.c b/drivers/gpu/drm/mxsfb/lcdif_kms.c index ade76c3f4e4e90..68b1eb7a8ff13b 100644 --- a/drivers/gpu/drm/mxsfb/lcdif_kms.c +++ b/drivers/gpu/drm/mxsfb/lcdif_kms.c @@ -374,14 +374,23 @@ static void lcdif_disable_controller(struct lcdif_drm_private *lcdif) int ret; reg = readl(lcdif->base + LCDC_V8_CTRLDESCL0_5); + /* Disable the layer for DMA. */ reg &= ~CTRLDESCL0_5_EN; + /* + * It is necessary to wait for the full frame to finish streaming + * through the DMA engine before we can safely disable it by removing + * the DISP_PARA_DISP_ON bit. Disabling it in-flight can leave the + * hardware confused and unable to resume streaming for the next frame. + */ + reg |= CTRLDESCL0_5_SHADOW_LOAD_EN; writel(reg, lcdif->base + LCDC_V8_CTRLDESCL0_5); + /* Wait for the frame to finish or timeout after 50 ms. */ ret = readl_poll_timeout(lcdif->base + LCDC_V8_CTRLDESCL0_5, - reg, !(reg & CTRLDESCL0_5_EN), - 0, 36000); /* Wait ~2 frame times max */ + reg, !(reg & CTRLDESCL0_5_SHADOW_LOAD_EN), + 200, 50000); if (ret) - drm_err(lcdif->drm, "Failed to disable controller!\n"); + drm_err(lcdif->drm, "Timed out waiting for final vblank!\n"); reg = readl(lcdif->base + LCDC_V8_DISP_PARA); reg &= ~DISP_PARA_DISP_ON; diff --git a/drivers/gpu/drm/omapdrm/dss/dsi.c b/drivers/gpu/drm/omapdrm/dss/dsi.c index 27fe7bca9e2cfd..70cfb779d6ae5d 100644 --- a/drivers/gpu/drm/omapdrm/dss/dsi.c +++ b/drivers/gpu/drm/omapdrm/dss/dsi.c @@ -455,15 +455,10 @@ static irqreturn_t omap_dsi_irq_handler(int irq, void *arg) timer_delete(&dsi->te_timer); #endif - /* make a copy and unlock, so that isrs can unregister - * themselves */ - memcpy(&dsi->isr_tables_copy, &dsi->isr_tables, - sizeof(dsi->isr_tables)); + dsi_handle_isrs(&dsi->isr_tables, irqstatus, vcstatus, ciostatus); spin_unlock(&dsi->irq_lock); - dsi_handle_isrs(&dsi->isr_tables_copy, irqstatus, vcstatus, ciostatus); - dsi_handle_irq_errors(dsi, irqstatus, vcstatus, ciostatus); dsi_collect_irq_stats(dsi, irqstatus, vcstatus, ciostatus); diff --git a/drivers/gpu/drm/omapdrm/dss/dsi.h b/drivers/gpu/drm/omapdrm/dss/dsi.h index 601707c0ecc4e8..2b25247ea8935d 100644 --- a/drivers/gpu/drm/omapdrm/dss/dsi.h +++ b/drivers/gpu/drm/omapdrm/dss/dsi.h @@ -379,8 +379,6 @@ struct dsi_data { spinlock_t irq_lock; struct dsi_isr_tables isr_tables; - /* space for a copy used by the interrupt handler */ - struct dsi_isr_tables isr_tables_copy; int update_vc; #ifdef DSI_PERF_MEASURE diff --git a/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c b/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c index 1b14aa4efe359a..54a65abf7e8907 100644 --- a/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c +++ b/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c @@ -88,16 +88,22 @@ static int s6d16d0_prepare(struct drm_panel *panel) MIPI_DSI_DCS_TEAR_MODE_VBLANK); if (ret) { dev_err(s6->dev, "failed to enable vblank TE (%d)\n", ret); - return ret; + goto err_power_off; } /* Exit sleep mode and power on */ ret = mipi_dsi_dcs_exit_sleep_mode(dsi); if (ret) { dev_err(s6->dev, "failed to exit sleep mode (%d)\n", ret); - return ret; + goto err_power_off; } return 0; + +err_power_off: + gpiod_set_value_cansleep(s6->reset_gpio, 1); + regulator_disable(s6->supply); + + return ret; } static int s6d16d0_enable(struct drm_panel *panel) diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c index e8dc4096c1d25d..74ec417115a3d5 100644 --- a/drivers/gpu/drm/panthor/panthor_drv.c +++ b/drivers/gpu/drm/panthor/panthor_drv.c @@ -1814,7 +1814,7 @@ static int panthor_probe(struct platform_device *pdev) ptdev = devm_drm_dev_alloc(&pdev->dev, &panthor_drm_driver, struct panthor_device, base); if (IS_ERR(ptdev)) - return -ENOMEM; + return PTR_ERR(ptdev); platform_set_drvdata(pdev, ptdev); diff --git a/drivers/gpu/drm/panthor/panthor_gem.c b/drivers/gpu/drm/panthor/panthor_gem.c index 54535bae2b0cfe..772121ece31806 100644 --- a/drivers/gpu/drm/panthor/panthor_gem.c +++ b/drivers/gpu/drm/panthor/panthor_gem.c @@ -1351,6 +1351,24 @@ err_free_kbo: return ERR_PTR(ret); } +/** + * panthor_dummy_bo_create() - Create a Panthor BO meant to back sparse bindings. + * @ptdev: Device. + * + * Return: A valid pointer in case of success, an ERR_PTR() otherwise. + */ +struct panthor_gem_object * +panthor_dummy_bo_create(struct panthor_device *ptdev) +{ + /* Since even when the DRM device's mount point has enabled THP we have no guarantee + * that drm_gem_get_pages() will return a single 2MiB PMD, and also we cannot be sure + * that the 2MiB won't be reclaimed and re-allocated later on as 4KiB chunks, it doesn't + * make sense to pre-populate this object's page array, nor to fall back on a BO size + * of 4KiB. Sticking to a dummy object size of 2MiB lets us keep things simple for now. + */ + return panthor_gem_create(&ptdev->base, SZ_2M, DRM_PANTHOR_BO_NO_MMAP, NULL, 0); +} + static bool can_swap(void) { return get_nr_swap_pages() > 0; diff --git a/drivers/gpu/drm/panthor/panthor_gem.h b/drivers/gpu/drm/panthor/panthor_gem.h index 56d63137b4ebbf..5ae37d0d3646f5 100644 --- a/drivers/gpu/drm/panthor/panthor_gem.h +++ b/drivers/gpu/drm/panthor/panthor_gem.h @@ -325,6 +325,8 @@ panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm, void panthor_kernel_bo_destroy(struct panthor_kernel_bo *bo); +struct panthor_gem_object *panthor_dummy_bo_create(struct panthor_device *ptdev); + #ifdef CONFIG_DEBUG_FS void panthor_gem_debugfs_init(struct drm_minor *minor); #endif diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c index e10dbd18d8cf9a..fba2fff94ce32d 100644 --- a/drivers/gpu/drm/panthor/panthor_mmu.c +++ b/drivers/gpu/drm/panthor/panthor_mmu.c @@ -116,6 +116,17 @@ struct panthor_mmu { struct panthor_vm_pool { /** @xa: Array used for VM handle tracking. */ struct xarray xa; + + /** + * @dummy: Dummy object used for sparse mappings + * + * Sparse bindings map virtual address ranges onto a dummy + * BO in a modulo fashion. Even though sparse writes are meant + * to be discarded and reads undefined, writes are still reflected + * in the dummy buffer. That means we must keep a dummy object per + * file context, to avoid data leaks between them. + */ + struct panthor_gem_object *dummy; }; /** @@ -307,6 +318,9 @@ struct panthor_vm { u64 end; } kernel_auto_va; + /** @user_va_range: Upper boundary of VAs VM users can map objects against. */ + u64 user_va_range; + /** @as: Address space related fields. */ struct { /** @@ -403,6 +417,15 @@ struct panthor_vm { */ struct list_head lru_node; } reclaim; + + /** + * @dummy: Dummy object used for sparse mappings. + * + * VM's must keep a reference to the file context-wide dummy BO because + * they can outlive the file context, which includes the VM pool holding + * the original dummy BO reference. + */ + struct panthor_gem_object *dummy; }; /** @@ -1035,6 +1058,30 @@ panthor_vm_map_pages(struct panthor_vm *vm, u64 iova, int prot, return 0; } +static int +panthor_vm_map_sparse(struct panthor_vm *vm, u64 iova, int prot, + struct sg_table *sgt, u64 size) +{ + u64 mapped = 0; + int ret; + + while (mapped < size) { + u64 addr = iova + mapped; + u32 chunk_size = min(size - mapped, SZ_2M - (addr & (SZ_2M - 1))); + + ret = panthor_vm_map_pages(vm, addr, prot, sgt, + addr % SZ_2M, chunk_size); + if (ret) { + panthor_vm_unmap_pages(vm, iova, mapped); + return ret; + } + + mapped += chunk_size; + } + + return 0; +} + static int flags_to_prot(u32 flags) { int prot = 0; @@ -1277,15 +1324,15 @@ static int panthor_vm_op_ctx_prealloc_pts(struct panthor_vm_op_ctx *op_ctx) (DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \ DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \ DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED | \ + DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE | \ DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx, struct panthor_vm *vm, struct panthor_gem_object *bo, - u64 offset, - u64 size, u64 va, - u32 flags) + const struct drm_panthor_vm_bind_op *op) { + bool is_sparse = op->flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE; struct drm_gpuvm_bo *preallocated_vm_bo; struct sg_table *sgt = NULL; int ret; @@ -1293,12 +1340,25 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx, if (!bo) return -EINVAL; - if ((flags & ~PANTHOR_VM_BIND_OP_MAP_FLAGS) || - (flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) != DRM_PANTHOR_VM_BIND_OP_TYPE_MAP) + if ((op->flags & ~PANTHOR_VM_BIND_OP_MAP_FLAGS) || + (op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) != DRM_PANTHOR_VM_BIND_OP_TYPE_MAP) + return -EINVAL; + + /* uAPI mandates sparsely bound regions must not be executable. */ + if (is_sparse && !(op->flags & DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC)) return -EINVAL; - /* Make sure the VA and size are in-bounds. */ - if (size > bo->base.size || offset > bo->base.size - size) + /* For non-sparse, make sure the VA and size are in-bounds. + * For sparse, this is not applicable, because the dummy BO is + * repeatedly mapped over a potentially wider VA range. + */ + if (!is_sparse && (op->size > bo->base.size || op->bo_offset > bo->base.size - op->size)) + return -EINVAL; + + /* For sparse, we don't expect any user BO, the BO we get passed + * is the dummy BO attached to the VM pool. + */ + if (is_sparse && (op->bo_handle || op->bo_offset)) return -EINVAL; /* If the BO has an exclusive VM attached, it can't be mapped to other VMs. */ @@ -1306,7 +1366,7 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx, bo->exclusive_vm_root_gem != panthor_vm_root_gem(vm)) return -EINVAL; - panthor_vm_init_op_ctx(op_ctx, size, va, flags); + panthor_vm_init_op_ctx(op_ctx, op->size, op->va, op->flags); ret = panthor_vm_op_ctx_prealloc_vmas(op_ctx); if (ret) @@ -1335,7 +1395,7 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx, } op_ctx->map.vm_bo = drm_gpuvm_bo_obtain_prealloc(preallocated_vm_bo); - op_ctx->map.bo_offset = offset; + op_ctx->map.bo_offset = op->bo_offset; ret = panthor_vm_op_ctx_prealloc_pts(op_ctx); if (ret) @@ -1446,7 +1506,9 @@ panthor_vm_get_bo_for_va(struct panthor_vm *vm, u64 va, u64 *bo_offset) if (vma && vma->base.gem.obj) { drm_gem_object_get(vma->base.gem.obj); bo = to_panthor_bo(vma->base.gem.obj); - *bo_offset = vma->base.gem.offset + (va - vma->base.va.addr); + *bo_offset = !(vma->flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE) ? + vma->base.gem.offset + (va - vma->base.va.addr) : + va & (SZ_2M - 1); } mutex_unlock(&vm->op_lock); @@ -1551,6 +1613,9 @@ int panthor_vm_pool_create_vm(struct panthor_device *ptdev, if (IS_ERR(vm)) return PTR_ERR(vm); + drm_gem_object_get(&pool->dummy->base); + vm->dummy = pool->dummy; + ret = xa_alloc(&pool->xa, &id, vm, XA_LIMIT(1, PANTHOR_MAX_VMS_PER_FILE), GFP_KERNEL); @@ -1650,6 +1715,8 @@ void panthor_vm_pool_destroy(struct panthor_file *pfile) xa_for_each(&pfile->vms->xa, i, vm) panthor_vm_destroy(vm); + if (pfile->vms->dummy) + drm_gem_object_put(&pfile->vms->dummy->base); xa_destroy(&pfile->vms->xa); kfree(pfile->vms); } @@ -1662,12 +1729,28 @@ void panthor_vm_pool_destroy(struct panthor_file *pfile) */ int panthor_vm_pool_create(struct panthor_file *pfile) { + struct panthor_gem_object *dummy; + int ret; + pfile->vms = kzalloc_obj(*pfile->vms); if (!pfile->vms) return -ENOMEM; xa_init_flags(&pfile->vms->xa, XA_FLAGS_ALLOC1); + + dummy = panthor_dummy_bo_create(pfile->ptdev); + if (IS_ERR(dummy)) { + ret = PTR_ERR(dummy); + goto err_destroy_vm_pool; + } + + pfile->vms->dummy = dummy; + return 0; + +err_destroy_vm_pool: + panthor_vm_pool_destroy(pfile); + return ret; } /* dummy TLB ops, the real TLB flush happens in panthor_vm_flush_range() */ @@ -2004,6 +2087,9 @@ static void panthor_vm_free(struct drm_gpuvm *gpuvm) free_io_pgtable_ops(vm->pgtbl_ops); + if (vm->dummy) + drm_gem_object_put(&vm->dummy->base); + drm_mm_takedown(&vm->mm); kfree(vm); } @@ -2163,7 +2249,30 @@ static void panthor_vma_init(struct panthor_vma *vma, u32 flags) #define PANTHOR_VM_MAP_FLAGS \ (DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \ DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \ - DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED) + DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED | \ + DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE) + +static void +panthor_fix_sparse_map_offset(struct drm_gpuva_op_map *op, u32 flags) +{ + if (op && (flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE)) + op->gem.offset = op->va.addr & (SZ_2M - 1); +} + +static int +panthor_vm_exec_map_op(struct panthor_vm *vm, u32 flags, + const struct drm_gpuva_op_map *op) +{ + struct panthor_gem_object *bo = to_panthor_bo(op->gem.obj); + int prot = flags_to_prot(flags); + + if (flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE) + return panthor_vm_map_sparse(vm, op->va.addr, prot, + bo->dmap.sgt, op->va.range); + + return panthor_vm_map_pages(vm, op->va.addr, prot, bo->dmap.sgt, + op->gem.offset, op->va.range); +} static int panthor_gpuva_sm_step_map(struct drm_gpuva_op *op, void *priv) { @@ -2176,10 +2285,9 @@ static int panthor_gpuva_sm_step_map(struct drm_gpuva_op *op, void *priv) return -EINVAL; panthor_vma_init(vma, op_ctx->flags & PANTHOR_VM_MAP_FLAGS); + panthor_fix_sparse_map_offset(&op->map, vma->flags); - ret = panthor_vm_map_pages(vm, op->map.va.addr, flags_to_prot(vma->flags), - op_ctx->map.bo->dmap.sgt, op->map.gem.offset, - op->map.va.range); + ret = panthor_vm_exec_map_op(vm, vma->flags, &op->map); if (ret) { panthor_vm_op_ctx_return_vma(op_ctx, vma); return ret; @@ -2211,6 +2319,8 @@ static void unmap_hugepage_align(const struct drm_gpuva_op_remap *op, u64 *unmap_start, u64 *unmap_range) { + struct panthor_vma *unmap_vma = container_of(op->unmap->va, struct panthor_vma, base); + bool is_sparse = unmap_vma->flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE; u64 aligned_unmap_start, aligned_unmap_end, unmap_end; unmap_end = *unmap_start + *unmap_range; @@ -2218,11 +2328,15 @@ unmap_hugepage_align(const struct drm_gpuva_op_remap *op, aligned_unmap_end = ALIGN(unmap_end, SZ_2M); /* If we're dealing with a huge page, make sure the unmap region is - * aligned on the start of the page. + * aligned on the start of the page. If the unmapped VMA stands for + * a sparse mapping, always assume the backing storage is a THP, since + * the overhead of unmapping 2MiB worth of 4KiB pages and remapping + * some of them is offset by the logic of working out whether it's + * the opposite case right below. This also holds true for op->next. */ if (op->prev && aligned_unmap_start < *unmap_start && op->prev->va.addr <= aligned_unmap_start && - iova_mapped_as_huge_page(op->prev, *unmap_start)) { + (is_sparse || iova_mapped_as_huge_page(op->prev, *unmap_start))) { *unmap_range += *unmap_start - aligned_unmap_start; *unmap_start = aligned_unmap_start; } @@ -2232,7 +2346,7 @@ unmap_hugepage_align(const struct drm_gpuva_op_remap *op, */ if (op->next && aligned_unmap_end > unmap_end && op->next->va.addr + op->next->va.range >= aligned_unmap_end && - iova_mapped_as_huge_page(op->next, unmap_end - 1)) { + (is_sparse || iova_mapped_as_huge_page(op->next, unmap_end - 1))) { *unmap_range += aligned_unmap_end - unmap_end; } } @@ -2249,33 +2363,43 @@ static int panthor_gpuva_sm_step_remap(struct drm_gpuva_op *op, drm_gpuva_op_remap_to_unmap_range(&op->remap, &unmap_start, &unmap_range); - /* - * ARM IOMMU page table management code disallows partial unmaps of huge pages, - * so when a partial unmap is requested, we must first unmap the entire huge - * page and then remap the difference between the huge page minus the requested - * unmap region. Calculating the right start address and range for the expanded - * unmap operation is the responsibility of the following function. + /* op->remap.prev's BO offset is always the same as the unmap va's, but + * that of op->remap.next must be adjusted so as to remain < SZ_2M */ - unmap_hugepage_align(&op->remap, &unmap_start, &unmap_range); + panthor_fix_sparse_map_offset(op->remap.next, unmap_vma->flags); - /* If the range changed, we might have to lock a wider region to guarantee - * atomicity. panthor_vm_lock_region() bails out early if the new region - * is already part of the locked region, so no need to do this check here. - */ if (!unmap_vma->evicted) { + /* + * ARM IOMMU page table management code disallows partial unmaps of huge pages, + * so when a partial unmap is requested, we must first unmap the entire huge + * page and then remap the difference between the huge page minus the requested + * unmap region. Calculating the right start address and range for the expanded + * unmap operation is the responsibility of the following function. + */ + unmap_hugepage_align(&op->remap, &unmap_start, &unmap_range); + + /* If the range changed, we might have to lock a wider region to guarantee + * atomicity. panthor_vm_lock_region() bails out early if the new region + * is already part of the locked region, so no need to do this check here. + */ panthor_vm_lock_region(vm, unmap_start, unmap_range); panthor_vm_unmap_pages(vm, unmap_start, unmap_range); } if (op->remap.prev) { - struct panthor_gem_object *bo = to_panthor_bo(op->remap.prev->gem.obj); u64 offset = op->remap.prev->gem.offset + unmap_start - op->remap.prev->va.addr; u64 size = op->remap.prev->va.addr + op->remap.prev->va.range - unmap_start; - if (!unmap_vma->evicted) { - ret = panthor_vm_map_pages(vm, unmap_start, - flags_to_prot(unmap_vma->flags), - bo->dmap.sgt, offset, size); + if (!unmap_vma->evicted && size > 0) { + struct drm_gpuva_op_map map_op = { + .va.addr = unmap_start, + .va.range = size, + .gem.obj = op->remap.prev->gem.obj, + .gem.offset = offset, + }; + panthor_fix_sparse_map_offset(&map_op, unmap_vma->flags); + + ret = panthor_vm_exec_map_op(vm, unmap_vma->flags, &map_op); if (ret) return ret; } @@ -2286,14 +2410,19 @@ static int panthor_gpuva_sm_step_remap(struct drm_gpuva_op *op, } if (op->remap.next) { - struct panthor_gem_object *bo = to_panthor_bo(op->remap.next->gem.obj); u64 addr = op->remap.next->va.addr; u64 size = unmap_start + unmap_range - op->remap.next->va.addr; - if (!unmap_vma->evicted) { - ret = panthor_vm_map_pages(vm, addr, flags_to_prot(unmap_vma->flags), - bo->dmap.sgt, op->remap.next->gem.offset, - size); + if (!unmap_vma->evicted && size > 0) { + struct drm_gpuva_op_map map_op = { + .va.addr = addr, + .va.range = size, + .gem.obj = op->remap.next->gem.obj, + .gem.offset = op->remap.next->gem.offset, + }; + panthor_fix_sparse_map_offset(&map_op, unmap_vma->flags); + + ret = panthor_vm_exec_map_op(vm, unmap_vma->flags, &map_op); if (ret) return ret; } @@ -2490,11 +2619,17 @@ static int remap_evicted_vma(struct drm_gpuvm_bo *vm_bo, ret = panthor_vm_lock_region(vm, evicted_vma->base.va.addr, evicted_vma->base.va.range); if (!ret) { - ret = panthor_vm_map_pages(vm, evicted_vma->base.va.addr, - flags_to_prot(evicted_vma->flags), - bo->dmap.sgt, - evicted_vma->base.gem.offset, - evicted_vma->base.va.range); + struct drm_gpuva_op_map map_op = { + .va.addr = evicted_vma->base.va.addr, + .va.range = evicted_vma->base.va.range, + .gem.obj = &bo->base, + .gem.offset = evicted_vma->base.gem.offset, + }; + if (evicted_vma->flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE) + drm_WARN_ON_ONCE(&vm->ptdev->base, map_op.gem.offset != + (map_op.va.addr & (SZ_2M - 1))); + + ret = panthor_vm_exec_map_op(vm, evicted_vma->flags, &map_op); if (!ret) evicted_vma->evicted = false; @@ -2769,6 +2904,8 @@ panthor_vm_create(struct panthor_device *ptdev, bool for_mcu, va_range = full_va_range; } + vm->user_va_range = kernel_va_start; + mutex_init(&vm->mm_lock); drm_mm_init(&vm->mm, kernel_va_start, kernel_va_size); vm->kernel_auto_va.start = auto_kernel_va_start; @@ -2857,15 +2994,22 @@ panthor_vm_bind_prepare_op_ctx(struct drm_file *file, if (!IS_ALIGNED(op->va | op->size | op->bo_offset, vm_pgsz)) return -EINVAL; + /* We don't allow mappings that overlap with kbo's reserved range */ + if (range_overflows(op->va, op->size, vm->user_va_range)) + return -EINVAL; + switch (op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) { case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP: - gem = drm_gem_object_lookup(file, op->bo_handle); + if (!(op->flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE)) { + gem = drm_gem_object_lookup(file, op->bo_handle); + } else { + gem = &vm->dummy->base; + drm_gem_object_get(&vm->dummy->base); + } + ret = panthor_vm_prepare_map_op_ctx(op_ctx, vm, gem ? to_panthor_bo(gem) : NULL, - op->bo_offset, - op->size, - op->va, - op->flags); + op); drm_gem_object_put(gem); return ret; @@ -3061,10 +3205,19 @@ int panthor_vm_bind_exec_sync_op(struct drm_file *file, int panthor_vm_map_bo_range(struct panthor_vm *vm, struct panthor_gem_object *bo, u64 offset, u64 size, u64 va, u32 flags) { + struct drm_panthor_vm_bind_op op = { + .bo_offset = offset, + .size = size, + .va = va, + .flags = flags, + }; struct panthor_vm_op_ctx op_ctx; int ret; - ret = panthor_vm_prepare_map_op_ctx(&op_ctx, vm, bo, offset, size, va, flags); + if (drm_WARN_ON(&vm->ptdev->base, flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE)) + return -EINVAL; + + ret = panthor_vm_prepare_map_op_ctx(&op_ctx, vm, bo, &op); if (ret) return ret; diff --git a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c index 06072efd7fca37..b1ed25cefe5ed2 100644 --- a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c +++ b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c @@ -8,6 +8,7 @@ * Jeff Chen */ +#include #include #include #include @@ -206,7 +207,6 @@ static void rockchip_dp_drm_encoder_enable(struct drm_encoder *encoder, struct drm_crtc *crtc; struct drm_crtc_state *old_crtc_state; struct of_endpoint endpoint; - struct device_node *remote_port, *remote_port_parent; char name[32]; u32 port_id; int ret; @@ -230,18 +230,22 @@ static void rockchip_dp_drm_encoder_enable(struct drm_encoder *encoder, if (ret < 0) return; - remote_port_parent = of_graph_get_remote_port_parent(endpoint.local_node); + struct device_node *remote_port_parent __free(device_node) = + of_graph_get_remote_port_parent(endpoint.local_node); if (remote_port_parent) { - if (of_get_child_by_name(remote_port_parent, "ports")) { - remote_port = of_graph_get_remote_port(endpoint.local_node); + struct device_node *ports __free(device_node) = + of_get_child_by_name(remote_port_parent, "ports"); + + if (ports) { + struct device_node *remote_port __free(device_node) = + of_graph_get_remote_port(endpoint.local_node); + of_property_read_u32(remote_port, "reg", &port_id); - of_node_put(remote_port); sprintf(name, "%s vp%d", remote_port_parent->full_name, port_id); } else { sprintf(name, "%s %s", remote_port_parent->full_name, endpoint.id ? "vopl" : "vopb"); } - of_node_put(remote_port_parent); DRM_DEV_DEBUG(dp->dev, "vop %s output to dp\n", (ret) ? "LIT" : "BIG"); } @@ -311,6 +315,7 @@ static int rockchip_dp_of_probe(struct rockchip_dp_device *dp) { struct device *dev = dp->dev; struct device_node *np = dev->of_node; + struct clk *clk; dp->grf = syscon_regmap_lookup_by_phandle(np, "rockchip,grf"); if (IS_ERR(dp->grf)) @@ -327,6 +332,11 @@ static int rockchip_dp_of_probe(struct rockchip_dp_device *dp) return dev_err_probe(dev, PTR_ERR(dp->pclk), "failed to get pclk property\n"); + clk = devm_clk_get_optional_enabled(dev, "hclk"); + if (IS_ERR(clk)) + return dev_err_probe(dev, PTR_ERR(clk), + "failed to get hclk property\n"); + dp->rst = devm_reset_control_get(dev, "dp"); if (IS_ERR(dp->rst)) return dev_err_probe(dev, PTR_ERR(dp->rst), diff --git a/drivers/gpu/drm/rockchip/dw_dp-rockchip.c b/drivers/gpu/drm/rockchip/dw_dp-rockchip.c index 32bc73a1d5e456..0de822360c8db9 100644 --- a/drivers/gpu/drm/rockchip/dw_dp-rockchip.c +++ b/drivers/gpu/drm/rockchip/dw_dp-rockchip.c @@ -107,15 +107,26 @@ static int dw_dp_rockchip_bind(struct device *dev, struct device *master, void * return PTR_ERR(dp->base); connector = drm_bridge_connector_init(drm_dev, encoder); - if (IS_ERR(connector)) + if (IS_ERR(connector)) { + dw_dp_unbind(dp->base); return dev_err_probe(dev, PTR_ERR(connector), - "Failed to init bridge connector"); + "Failed to init bridge connector\n"); + } return 0; } +static void dw_dp_rockchip_unbind(struct device *dev, struct device *master, + void *data) +{ + struct rockchip_dw_dp *dp = dev_get_drvdata(dev); + + dw_dp_unbind(dp->base); +} + static const struct component_ops dw_dp_rockchip_component_ops = { .bind = dw_dp_rockchip_bind, + .unbind = dw_dp_rockchip_unbind, }; static int dw_dp_probe(struct platform_device *pdev) diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c index a160077a507f23..a268bd2fbaaaba 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c @@ -337,7 +337,8 @@ static bool vop2_output_uv_swap(u32 bus_format, u32 output_mode) static bool vop2_output_rg_swap(struct vop2 *vop2, u32 bus_format) { - if (vop2->version == VOP_VERSION_RK3588) { + if (vop2->version == VOP_VERSION_RK3588 || + vop2->version == VOP_VERSION_RK3576) { if (bus_format == MEDIA_BUS_FMT_YUV8_1X24 || bus_format == MEDIA_BUS_FMT_YUV10_1X30) return true; @@ -351,6 +352,8 @@ static bool is_yuv_output(u32 bus_format) switch (bus_format) { case MEDIA_BUS_FMT_YUV8_1X24: case MEDIA_BUS_FMT_YUV10_1X30: + case MEDIA_BUS_FMT_YUYV10_1X20: + case MEDIA_BUS_FMT_UYVY10_1X20: case MEDIA_BUS_FMT_UYYVYY8_0_5X24: case MEDIA_BUS_FMT_UYYVYY10_0_5X30: case MEDIA_BUS_FMT_YUYV8_2X8: diff --git a/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c index 17eda592b1833a..322a303d3f1a53 100644 --- a/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c +++ b/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c @@ -2288,15 +2288,6 @@ static void rk3568_vop2_setup_layer_mixer(struct vop2_video_port *vp) * lead to the configuration of the previous VP being take effect along with the VSYNC * of the new VP. */ - if (layer_sel != old_layer_sel || port_sel != old_port_sel) - ovl_ctrl |= FIELD_PREP(RK3568_OVL_CTRL__LAYERSEL_REGDONE_SEL, vp->id); - vop2_writel(vop2, RK3568_OVL_CTRL, ovl_ctrl); - - if (port_sel != old_port_sel) { - vop2_writel(vop2, RK3568_OVL_PORT_SEL, port_sel); - vop2_cfg_done(vp); - rk3568_vop2_wait_for_port_mux_done(vop2); - } if (layer_sel != old_layer_sel && atv_layer_sel != old_layer_sel) { cfg_done = vop2_readl(vop2, RK3568_REG_CFG_DONE); @@ -2306,7 +2297,17 @@ static void rk3568_vop2_setup_layer_mixer(struct vop2_video_port *vp) * Changes of other VPs' overlays have not taken effect */ if (cfg_done) - rk3568_vop2_wait_for_layer_cfg_done(vop2, vop2->old_layer_sel); + rk3568_vop2_wait_for_layer_cfg_done(vop2, old_layer_sel); + } + + if (layer_sel != old_layer_sel || port_sel != old_port_sel) + ovl_ctrl |= FIELD_PREP(RK3568_OVL_CTRL__LAYERSEL_REGDONE_SEL, vp->id); + vop2_writel(vop2, RK3568_OVL_CTRL, ovl_ctrl); + + if (port_sel != old_port_sel) { + vop2_writel(vop2, RK3568_OVL_PORT_SEL, port_sel); + vop2_cfg_done(vp); + rk3568_vop2_wait_for_port_mux_done(vop2); } vop2_writel(vop2, RK3568_OVL_LAYER_SEL, layer_sel); diff --git a/drivers/gpu/drm/solomon/ssd130x.c b/drivers/gpu/drm/solomon/ssd130x.c index 784f7000fad268..df67a1759bb3c3 100644 --- a/drivers/gpu/drm/solomon/ssd130x.c +++ b/drivers/gpu/drm/solomon/ssd130x.c @@ -1008,7 +1008,6 @@ static int ssd130x_fb_blit_rect(struct drm_framebuffer *fb, struct ssd130x_device *ssd130x = drm_to_ssd130x(fb->dev); struct iosys_map dst; unsigned int dst_pitch; - int ret = 0; /* Align y to display page boundaries */ rect->y1 = round_down(rect->y1, SSD130X_PAGE_HEIGHT); @@ -1021,7 +1020,7 @@ static int ssd130x_fb_blit_rect(struct drm_framebuffer *fb, ssd130x_update_rect(ssd130x, rect, buf, data_array); - return ret; + return 0; } static int ssd132x_fb_blit_rect(struct drm_framebuffer *fb, @@ -1033,7 +1032,6 @@ static int ssd132x_fb_blit_rect(struct drm_framebuffer *fb, struct ssd130x_device *ssd130x = drm_to_ssd130x(fb->dev); unsigned int dst_pitch; struct iosys_map dst; - int ret = 0; /* Align x to display segment boundaries */ rect->x1 = round_down(rect->x1, SSD132X_SEGMENT_WIDTH); @@ -1047,7 +1045,7 @@ static int ssd132x_fb_blit_rect(struct drm_framebuffer *fb, ssd132x_update_rect(ssd130x, rect, buf, data_array); - return ret; + return 0; } static int ssd133x_fb_blit_rect(struct drm_framebuffer *fb, @@ -1059,7 +1057,6 @@ static int ssd133x_fb_blit_rect(struct drm_framebuffer *fb, const struct drm_format_info *fi = drm_format_info(DRM_FORMAT_RGB332); unsigned int dst_pitch; struct iosys_map dst; - int ret = 0; if (!fi) return -EINVAL; @@ -1071,7 +1068,7 @@ static int ssd133x_fb_blit_rect(struct drm_framebuffer *fb, ssd133x_update_rect(ssd130x, rect, data_array, dst_pitch); - return ret; + return 0; } static int ssd130x_primary_plane_atomic_check(struct drm_plane *plane, diff --git a/drivers/gpu/drm/sun4i/sun4i_crtc.c b/drivers/gpu/drm/sun4i/sun4i_crtc.c index c2f7f69baf75a3..3fa1f4fda0f0d0 100644 --- a/drivers/gpu/drm/sun4i/sun4i_crtc.c +++ b/drivers/gpu/drm/sun4i/sun4i_crtc.c @@ -208,7 +208,7 @@ struct sun4i_crtc *sun4i_crtc_init(struct drm_device *drm, planes = sunxi_engine_layers_init(drm, engine); if (IS_ERR(planes)) { dev_err(drm->dev, "Couldn't create the planes\n"); - return NULL; + return ERR_CAST(planes); } /* find primary and cursor planes for drm_crtc_init_with_planes */ diff --git a/drivers/gpu/drm/sun4i/sun4i_drv.c b/drivers/gpu/drm/sun4i/sun4i_drv.c index 8a409eee1dca53..e9baca360160b8 100644 --- a/drivers/gpu/drm/sun4i/sun4i_drv.c +++ b/drivers/gpu/drm/sun4i/sun4i_drv.c @@ -289,6 +289,8 @@ static void sun4i_drv_traverse_endpoints(struct endpoint_list *list, kfifo_put(&list->fifo, remote); } + + of_node_put(port); } static int sun4i_drv_add_endpoints(struct device *dev, @@ -394,6 +396,7 @@ static int sun4i_drv_probe(struct platform_device *pdev) /* process this endpoint */ ret = sun4i_drv_add_endpoints(&pdev->dev, &list, &match, endpoint); + of_node_put(endpoint); /* sun4i_drv_add_endpoints can fail to allocate memory */ if (ret < 0) diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c index 74c7c3720ba82f..a27d394d057087 100644 --- a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c +++ b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c @@ -161,7 +161,7 @@ static void sun4i_hdmi_enable(struct drm_encoder *encoder, drm_atomic_helper_connector_hdmi_update_infoframes(connector, state); - val |= SUN4I_HDMI_PKT_CTRL_TYPE(0, SUN4I_HDMI_PKT_AVI); + val = SUN4I_HDMI_PKT_CTRL_TYPE(0, SUN4I_HDMI_PKT_AVI); val |= SUN4I_HDMI_PKT_CTRL_TYPE(1, SUN4I_HDMI_PKT_END); writel(val, hdmi->base + SUN4I_HDMI_PKT_CTRL_REG(0)); diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.c b/drivers/gpu/drm/sun4i/sun4i_tcon.c index 960e83c8291da3..67047e3b657ff8 100644 --- a/drivers/gpu/drm/sun4i/sun4i_tcon.c +++ b/drivers/gpu/drm/sun4i/sun4i_tcon.c @@ -717,9 +717,11 @@ void sun4i_tcon_mode_set(struct sun4i_tcon *tcon, case DRM_MODE_ENCODER_DSI: /* DSI is tied to special case of CPU interface */ sun4i_tcon0_mode_set_cpu(tcon, encoder, mode); + sun4i_tcon_set_mux(tcon, 0, encoder); break; case DRM_MODE_ENCODER_LVDS: sun4i_tcon0_mode_set_lvds(tcon, encoder, mode); + sun4i_tcon_set_mux(tcon, 0, encoder); break; case DRM_MODE_ENCODER_NONE: sun4i_tcon0_mode_set_rgb(tcon, encoder, mode); @@ -970,6 +972,7 @@ static int sun4i_tcon_of_get_id_from_port(struct device_node *port) continue; ret = of_property_read_u32(remote, "reg", ®); + of_node_put(remote); if (ret) continue; @@ -1407,7 +1410,7 @@ static int sun8i_r40_tcon_tv_set_mux(struct sun4i_tcon *tcon, { struct device_node *port, *remote; struct platform_device *pdev; - int id, ret; + int id, ret = 0; /* find TCON TOP platform device and TCON id */ @@ -1430,21 +1433,20 @@ static int sun8i_r40_tcon_tv_set_mux(struct sun4i_tcon *tcon, if (IS_ENABLED(CONFIG_DRM_SUN8I_TCON_TOP) && encoder->encoder_type == DRM_MODE_ENCODER_TMDS) { ret = sun8i_tcon_top_set_hdmi_src(&pdev->dev, id); - if (ret) { - put_device(&pdev->dev); - return ret; - } + if (ret) + goto out_put_device; } if (IS_ENABLED(CONFIG_DRM_SUN8I_TCON_TOP)) { ret = sun8i_tcon_top_de_config(&pdev->dev, tcon->id, id); - if (ret) { - put_device(&pdev->dev); - return ret; - } + if (ret) + goto out_put_device; } - return 0; +out_put_device: + put_device(&pdev->dev); + + return ret; } static const struct sun4i_tcon_quirks sun4i_a10_quirks = { diff --git a/drivers/gpu/drm/sun4i/sun8i_dw_hdmi.c b/drivers/gpu/drm/sun4i/sun8i_dw_hdmi.c index 96532709c2a7ed..51cdfa1ae3bf48 100644 --- a/drivers/gpu/drm/sun4i/sun8i_dw_hdmi.c +++ b/drivers/gpu/drm/sun4i/sun8i_dw_hdmi.c @@ -83,6 +83,8 @@ static u32 sun8i_dw_hdmi_find_possible_crtcs(struct drm_device *drm, of_node_put(remote_port); } } + + of_node_put(port); } else { crtcs = drm_of_find_possible_crtcs(drm, node); } diff --git a/drivers/gpu/drm/sun4i/sun8i_hdmi_phy.c b/drivers/gpu/drm/sun4i/sun8i_hdmi_phy.c index 4fa69c463dc463..489ea94693ffa8 100644 --- a/drivers/gpu/drm/sun4i/sun8i_hdmi_phy.c +++ b/drivers/gpu/drm/sun4i/sun8i_hdmi_phy.c @@ -91,7 +91,7 @@ static const struct dw_hdmi_mpll_config sun50i_h6_mpll_cfg[] = { }, }, { 594000000, { - { 0x1a40, 0x0003 }, + { 0x1a7c, 0x0003 }, { 0x3b4c, 0x0003 }, { 0x5a64, 0x0003 }, }, diff --git a/drivers/gpu/drm/sun4i/sun8i_mixer.c b/drivers/gpu/drm/sun4i/sun8i_mixer.c index 4e42262950104b..b6cd7352e76c56 100644 --- a/drivers/gpu/drm/sun4i/sun8i_mixer.c +++ b/drivers/gpu/drm/sun4i/sun8i_mixer.c @@ -785,7 +785,7 @@ static const struct sun8i_mixer_cfg sun8i_v3s_mixer_cfg = { .de_type = SUN8I_MIXER_DE2, .vi_scaler_num = 2, .scaler_mask = 0x3, - .scanline_yuv = 2048, + .scanline_yuv = 1024, }, .de_type = SUN8I_MIXER_DE2, .mod_rate = 150000000, diff --git a/drivers/gpu/drm/sun4i/sun8i_tcon_top.c b/drivers/gpu/drm/sun4i/sun8i_tcon_top.c index 8adda578c51ba4..9cbd655518b20a 100644 --- a/drivers/gpu/drm/sun4i/sun8i_tcon_top.c +++ b/drivers/gpu/drm/sun4i/sun8i_tcon_top.c @@ -25,6 +25,49 @@ static bool sun8i_tcon_top_node_is_tcon_top(struct device_node *node) return !!of_match_node(sun8i_tcon_top_of_table, node); } +static unsigned int sun8i_tcon_top_get_tcon_map(struct device_node *node) +{ + static const u32 out_ports[] = { + TCON_TOP_MIXER0_OUT_PORT, + TCON_TOP_MIXER1_OUT_PORT, + }; + unsigned int i, map = 0; + + for (i = 0; i < ARRAY_SIZE(out_ports); i++) { + struct device_node *port; + + port = of_graph_get_port_by_id(node, out_ports[i]); + if (!port) + continue; + + for_each_of_graph_port_endpoint(port, ep) { + struct of_endpoint endpoint; + + if (of_graph_parse_endpoint(ep, &endpoint)) + continue; + + if (endpoint.id < TCON_TOP_PORT_TCON_NUM) + map |= BIT(endpoint.id); + } + + of_node_put(port); + } + + return map; +} + +static unsigned int sun8i_tcon_top_park_index(struct sun8i_tcon_top *tcon_top, + int tcon) +{ + unsigned int candidates; + + candidates = tcon_top->tcon_map & ~BIT(tcon); + if (!candidates) + candidates = GENMASK(TCON_TOP_PORT_TCON_NUM - 1, 0) & ~BIT(tcon); + + return ffs(candidates) - 1; +} + int sun8i_tcon_top_set_hdmi_src(struct device *dev, int tcon) { struct sun8i_tcon_top *tcon_top = dev_get_drvdata(dev); @@ -57,6 +100,7 @@ EXPORT_SYMBOL(sun8i_tcon_top_set_hdmi_src); int sun8i_tcon_top_de_config(struct device *dev, int mixer, int tcon) { struct sun8i_tcon_top *tcon_top = dev_get_drvdata(dev); + u32 mixer_msk, other_msk; unsigned long flags; u32 reg; @@ -70,21 +114,27 @@ int sun8i_tcon_top_de_config(struct device *dev, int mixer, int tcon) return -EINVAL; } - if (tcon > 3) { - dev_err(dev, "TCON index is too high!\n"); + if (tcon < 0 || tcon >= TCON_TOP_PORT_TCON_NUM) { + dev_err(dev, "TCON index is invalid!\n"); return -EINVAL; } + mixer_msk = mixer ? TCON_TOP_PORT_DE1_MSK : TCON_TOP_PORT_DE0_MSK; + other_msk = mixer ? TCON_TOP_PORT_DE0_MSK : TCON_TOP_PORT_DE1_MSK; + spin_lock_irqsave(&tcon_top->reg_lock, flags); reg = readl(tcon_top->regs + TCON_TOP_PORT_SEL_REG); - if (mixer == 0) { - reg &= ~TCON_TOP_PORT_DE0_MSK; - reg |= FIELD_PREP(TCON_TOP_PORT_DE0_MSK, tcon); - } else { - reg &= ~TCON_TOP_PORT_DE1_MSK; - reg |= FIELD_PREP(TCON_TOP_PORT_DE1_MSK, tcon); + + reg &= ~mixer_msk; + reg |= field_prep(mixer_msk, tcon); + + if (field_get(other_msk, reg) == tcon) { + reg &= ~other_msk; + reg |= field_prep(other_msk, + sun8i_tcon_top_park_index(tcon_top, tcon)); } + writel(reg, tcon_top->regs + TCON_TOP_PORT_SEL_REG); spin_unlock_irqrestore(&tcon_top->reg_lock, flags); @@ -143,6 +193,7 @@ static int sun8i_tcon_top_bind(struct device *dev, struct device *master, return -ENOMEM; clk_data->num = CLK_NUM; tcon_top->clk_data = clk_data; + tcon_top->tcon_map = sun8i_tcon_top_get_tcon_map(dev->of_node); spin_lock_init(&tcon_top->reg_lock); @@ -175,10 +226,6 @@ static int sun8i_tcon_top_bind(struct device *dev, struct device *master, goto err_assert_reset; } - /* - * At least on H6, some registers have some bits set by default - * which may cause issues. Clear them here. - */ writel(0, regs + TCON_TOP_PORT_SEL_REG); writel(0, regs + TCON_TOP_GATE_SRC_REG); diff --git a/drivers/gpu/drm/sun4i/sun8i_tcon_top.h b/drivers/gpu/drm/sun4i/sun8i_tcon_top.h index 0390584a330e50..2b887470a49f8b 100644 --- a/drivers/gpu/drm/sun4i/sun8i_tcon_top.h +++ b/drivers/gpu/drm/sun4i/sun8i_tcon_top.h @@ -14,6 +14,10 @@ #define TCON_TOP_PORT_SEL_REG 0x1C #define TCON_TOP_PORT_DE0_MSK GENMASK(1, 0) #define TCON_TOP_PORT_DE1_MSK GENMASK(5, 4) +#define TCON_TOP_PORT_TCON_NUM 4 + +#define TCON_TOP_MIXER0_OUT_PORT 1 +#define TCON_TOP_MIXER1_OUT_PORT 3 #define TCON_TOP_GATE_SRC_REG 0x20 #define TCON_TOP_HDMI_SRC_MSK GENMASK(29, 28) @@ -29,6 +33,8 @@ struct sun8i_tcon_top { void __iomem *regs; struct reset_control *rst; + unsigned int tcon_map; + /* * spinlock is used to synchronize access to same * register where multiple clock gates can be set. diff --git a/drivers/gpu/drm/sun4i/sun8i_vi_scaler.c b/drivers/gpu/drm/sun4i/sun8i_vi_scaler.c index 3dec4eeb1ba21d..a5cc99981006aa 100644 --- a/drivers/gpu/drm/sun4i/sun8i_vi_scaler.c +++ b/drivers/gpu/drm/sun4i/sun8i_vi_scaler.c @@ -894,20 +894,26 @@ static void sun8i_vi_scaler_set_coeff(struct regmap *map, u32 base, lan3coefftab32_left[offset + i]); regmap_write(map, SUN8I_SCALER_VSU_YHCOEFF1(base, i), lan3coefftab32_right[offset + i]); + } + offset = sun8i_vi_scaler_coef_index(vstep) * + SUN8I_VI_SCALER_COEFF_COUNT; + for (i = 0; i < SUN8I_VI_SCALER_COEFF_COUNT; i++) + regmap_write(map, SUN8I_SCALER_VSU_YVCOEFF(base, i), + lan2coefftab32[offset + i]); + + offset = sun8i_vi_scaler_coef_index(hstep / format->hsub) * + SUN8I_VI_SCALER_COEFF_COUNT; + for (i = 0; i < SUN8I_VI_SCALER_COEFF_COUNT; i++) { regmap_write(map, SUN8I_SCALER_VSU_CHCOEFF0(base, i), ch_left[offset + i]); regmap_write(map, SUN8I_SCALER_VSU_CHCOEFF1(base, i), ch_right[offset + i]); } - - offset = sun8i_vi_scaler_coef_index(hstep) * + offset = sun8i_vi_scaler_coef_index(vstep / format->vsub) * SUN8I_VI_SCALER_COEFF_COUNT; - for (i = 0; i < SUN8I_VI_SCALER_COEFF_COUNT; i++) { - regmap_write(map, SUN8I_SCALER_VSU_YVCOEFF(base, i), - lan2coefftab32[offset + i]); + for (i = 0; i < SUN8I_VI_SCALER_COEFF_COUNT; i++) regmap_write(map, SUN8I_SCALER_VSU_CVCOEFF(base, i), cy[offset + i]); - } } void sun8i_vi_scaler_enable(struct sun8i_layer *layer, bool enable) @@ -969,6 +975,8 @@ void sun8i_vi_scaler_setup(struct sun8i_layer *layer, regmap_write(layer->regs, SUN50I_SCALER_VSU_SCALE_MODE(base), val); + regmap_write(layer->regs, + SUN50I_SCALER_VSU_GLB_ALPHA(base), 0xff); } regmap_write(layer->regs, diff --git a/drivers/gpu/drm/sun4i/sun8i_vi_scaler.h b/drivers/gpu/drm/sun4i/sun8i_vi_scaler.h index 245fe2f431c3c1..d89c0cbe188dd0 100644 --- a/drivers/gpu/drm/sun4i/sun8i_vi_scaler.h +++ b/drivers/gpu/drm/sun4i/sun8i_vi_scaler.h @@ -36,6 +36,7 @@ #define SUN50I_SCALER_VSU_EDSCL_CTRL(base) ((base) + 0x28) #define SUN50I_SCALER_VSU_ANGLE_THR(base) ((base) + 0x2c) #define SUN8I_SCALER_VSU_OUTSIZE(base) ((base) + 0x40) +#define SUN50I_SCALER_VSU_GLB_ALPHA(base) ((base) + 0x44) #define SUN8I_SCALER_VSU_YINSIZE(base) ((base) + 0x80) #define SUN8I_SCALER_VSU_YHSTEP(base) ((base) + 0x88) #define SUN8I_SCALER_VSU_YVSTEP(base) ((base) + 0x8c) diff --git a/drivers/gpu/drm/tegra/dsi.c b/drivers/gpu/drm/tegra/dsi.c index 7f25c50621c94e..690e4488de3f6e 100644 --- a/drivers/gpu/drm/tegra/dsi.c +++ b/drivers/gpu/drm/tegra/dsi.c @@ -922,6 +922,15 @@ static void tegra_dsi_encoder_enable(struct drm_encoder *encoder) return; } + /* If the bootloader enabled DSI it needs to be disabled + * in order for the panel initialization commands to be + * properly sent. + */ + value = tegra_dsi_readl(dsi, DSI_POWER_CONTROL); + + if (value & DSI_POWER_CONTROL_ENABLE) + tegra_dsi_disable(dsi); + state = tegra_dsi_get_state(dsi); tegra_dsi_set_timeout(dsi, state->bclk, state->vrefresh); diff --git a/drivers/gpu/drm/tve200/tve200_drv.c b/drivers/gpu/drm/tve200/tve200_drv.c index 562f3f11812a39..f5ef468538f9b6 100644 --- a/drivers/gpu/drm/tve200/tve200_drv.c +++ b/drivers/gpu/drm/tve200/tve200_drv.c @@ -263,6 +263,7 @@ static const struct of_device_id tve200_of_match[] = { }, {}, }; +MODULE_DEVICE_TABLE(of, tve200_of_match); static struct platform_driver tve200_driver = { .driver = { diff --git a/drivers/gpu/drm/v3d/Kconfig b/drivers/gpu/drm/v3d/Kconfig index ce62c5908e1db6..6a33e0ab30de6e 100644 --- a/drivers/gpu/drm/v3d/Kconfig +++ b/drivers/gpu/drm/v3d/Kconfig @@ -5,6 +5,7 @@ config DRM_V3D depends on DRM depends on COMMON_CLK depends on MMU + select DRM_EXEC select DRM_SCHED select DRM_GEM_SHMEM_HELPER help diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h index 8779a42c65081a..c64ba176b81cf2 100644 --- a/drivers/gpu/drm/v3d/v3d_drv.h +++ b/drivers/gpu/drm/v3d/v3d_drv.h @@ -7,7 +7,8 @@ #include #include -#include +#include +#include #include #include #include @@ -294,6 +295,27 @@ to_v3d_fence(struct dma_fence *fence) #define V3D_CORE_READ(core, offset) readl(v3d->core_regs[core] + offset) #define V3D_CORE_WRITE(core, offset, val) writel(val, v3d->core_regs[core] + offset) +#define V3D_MAX_JOBS_PER_SUBMISSION 3 + +/* Per-ioctl submission context */ +struct v3d_submit { + struct v3d_dev *v3d; + + struct drm_file *file_priv; + + /* DRM exec context for this submission. */ + struct drm_exec exec; + + /* Ordered array of jobs forming the submission chain. Jobs are + * appended via v3d_submit_add_job(), then chained and pushed to + * the scheduler by v3d_submit_jobs(). + */ + struct v3d_job *jobs[V3D_MAX_JOBS_PER_SUBMISSION]; + + /* Number of jobs currently in @jobs. */ + u32 job_count; +}; + struct v3d_job { struct drm_sched_job base; @@ -301,6 +323,9 @@ struct v3d_job { struct v3d_dev *v3d; + /* The queue that the job was submitted on. */ + enum v3d_queue queue; + /* This is the array of BOs that were looked up at the start * of submission. */ @@ -333,6 +358,11 @@ struct v3d_job { void (*free)(struct kref *ref); bool has_pm_ref; + + /* Whether the job needs implicit dependencies, i.e. must wait for + * other contexts still writing its BOs. + */ + bool has_implicit_dep; }; struct v3d_bin_job { @@ -410,6 +440,11 @@ struct v3d_indirect_csd_info { /* Clean cache job associated to the Indirect CSD job */ struct v3d_job *clean_job; + /* Indirect CSD args, stashed by the extension parser and later used + * to create the CSD job from them. + */ + struct drm_v3d_submit_csd args; + /* Offset within the BO where the workgroup counts are stored */ u32 offset; @@ -425,7 +460,7 @@ struct v3d_indirect_csd_info { struct drm_gem_object *indirect; /* Context of the Indirect CSD job */ - struct ww_acquire_ctx acquire_ctx; + struct drm_exec exec; }; struct v3d_timestamp_query_info { diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index bea46298b69e8f..663561ee8d8187 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -189,15 +189,11 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job) struct v3d_dev *v3d = job->base.v3d; struct v3d_queue_state *queue = &v3d->queue[V3D_BIN]; struct drm_device *dev = &v3d->drm; - struct dma_fence *fence; + struct dma_fence *fence = NULL; unsigned long irqflags; - if (unlikely(job->base.base.s_fence->finished.error)) { - spin_lock_irqsave(&queue->queue_lock, irqflags); - queue->active_job = NULL; - spin_unlock_irqrestore(&queue->queue_lock, irqflags); - return NULL; - } + if (unlikely(job->base.base.s_fence->finished.error)) + goto out_clean_job; /* Lock required around bin_job update vs * v3d_overflow_mem_work(). @@ -214,7 +210,7 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job) fence = v3d_fence_create(v3d, V3D_BIN); if (IS_ERR(fence)) - return NULL; + goto out_clean_job; if (job->base.irq_fence) dma_fence_put(job->base.irq_fence); @@ -242,6 +238,12 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job) V3D_CORE_WRITE(0, V3D_CLE_CT0QEA, job->end); return fence; + +out_clean_job: + spin_lock_irqsave(&queue->queue_lock, irqflags); + queue->active_job = NULL; + spin_unlock_irqrestore(&queue->queue_lock, irqflags); + return fence; } static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job) @@ -249,12 +251,10 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job) struct v3d_render_job *job = to_render_job(sched_job); struct v3d_dev *v3d = job->base.v3d; struct drm_device *dev = &v3d->drm; - struct dma_fence *fence; + struct dma_fence *fence = NULL; - if (unlikely(job->base.base.s_fence->finished.error)) { - v3d->queue[V3D_RENDER].active_job = NULL; - return NULL; - } + if (unlikely(job->base.base.s_fence->finished.error)) + goto out_clean_job; v3d->queue[V3D_RENDER].active_job = &job->base; @@ -268,7 +268,7 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job) fence = v3d_fence_create(v3d, V3D_RENDER); if (IS_ERR(fence)) - return NULL; + goto out_clean_job; if (job->base.irq_fence) dma_fence_put(job->base.irq_fence); @@ -289,6 +289,10 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job) V3D_CORE_WRITE(0, V3D_CLE_CT1QEA, job->end); return fence; + +out_clean_job: + v3d->queue[V3D_RENDER].active_job = NULL; + return fence; } static struct dma_fence * @@ -297,18 +301,16 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job) struct v3d_tfu_job *job = to_tfu_job(sched_job); struct v3d_dev *v3d = job->base.v3d; struct drm_device *dev = &v3d->drm; - struct dma_fence *fence; + struct dma_fence *fence = NULL; - if (unlikely(job->base.base.s_fence->finished.error)) { - v3d->queue[V3D_TFU].active_job = NULL; - return NULL; - } + if (unlikely(job->base.base.s_fence->finished.error)) + goto out_clean_job; v3d->queue[V3D_TFU].active_job = &job->base; fence = v3d_fence_create(v3d, V3D_TFU); if (IS_ERR(fence)) - return NULL; + goto out_clean_job; if (job->base.irq_fence) dma_fence_put(job->base.irq_fence); @@ -336,6 +338,10 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job) V3D_WRITE(V3D_TFU_ICFG(v3d->ver), job->args.icfg | V3D_TFU_ICFG_IOC); return fence; + +out_clean_job: + v3d->queue[V3D_TFU].active_job = NULL; + return fence; } static struct dma_fence * @@ -344,13 +350,11 @@ v3d_csd_job_run(struct drm_sched_job *sched_job) struct v3d_csd_job *job = to_csd_job(sched_job); struct v3d_dev *v3d = job->base.v3d; struct drm_device *dev = &v3d->drm; - struct dma_fence *fence; + struct dma_fence *fence = NULL; int i, csd_cfg0_reg; - if (unlikely(job->base.base.s_fence->finished.error)) { - v3d->queue[V3D_CSD].active_job = NULL; - return NULL; - } + if (unlikely(job->base.base.s_fence->finished.error)) + goto out_clean_job; /* The HW interprets a workgroup size of 0 as 65536; however, the * user-space driver exposes a maximum of 65535. Therefore, a 0 in @@ -368,7 +372,7 @@ v3d_csd_job_run(struct drm_sched_job *sched_job) fence = v3d_fence_create(v3d, V3D_CSD); if (IS_ERR(fence)) - return NULL; + goto out_clean_job; if (job->base.irq_fence) dma_fence_put(job->base.irq_fence); @@ -395,6 +399,10 @@ v3d_csd_job_run(struct drm_sched_job *sched_job) V3D_CORE_WRITE(0, csd_cfg0_reg, job->args.cfg[0]); return fence; + +out_clean_job: + v3d->queue[V3D_CSD].active_job = NULL; + return fence; } static void diff --git a/drivers/gpu/drm/v3d/v3d_submit.c b/drivers/gpu/drm/v3d/v3d_submit.c index 7682b24f13ec51..fdf27d7205877e 100644 --- a/drivers/gpu/drm/v3d/v3d_submit.c +++ b/drivers/gpu/drm/v3d/v3d_submit.c @@ -20,37 +20,57 @@ * to v3d, so we don't attach dma-buf fences to them. */ static int -v3d_lock_bo_reservations(struct v3d_job *job, - struct ww_acquire_ctx *acquire_ctx) +v3d_submit_lock_reservations(struct v3d_submit *submit) { - int i, ret; - - ret = drm_gem_lock_reservations(job->bo, job->bo_count, acquire_ctx); - if (ret) - return ret; - - for (i = 0; i < job->bo_count; i++) { - ret = dma_resv_reserve_fences(job->bo[i]->resv, 1); + int i, j, ret; + + drm_exec_init(&submit->exec, + DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES, 0); + drm_exec_until_all_locked(&submit->exec) { + for (i = 0; i < submit->job_count; i++) { + struct v3d_job *job = submit->jobs[i]; + + ret = drm_exec_prepare_array(&submit->exec, job->bo, + job->bo_count, 1); + if (ret) + break; + } + drm_exec_retry_on_contention(&submit->exec); if (ret) goto fail; + } - ret = drm_sched_job_add_implicit_dependencies(&job->base, - job->bo[i], true); - if (ret) - goto fail; + for (i = 0; i < submit->job_count; i++) { + struct v3d_job *job = submit->jobs[i]; + + if (!job->has_implicit_dep) + continue; + + for (j = 0; j < job->bo_count; j++) { + ret = drm_sched_job_add_implicit_dependencies(&job->base, + job->bo[j], + true); + if (ret) + goto fail; + } } return 0; fail: - drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx); + drm_exec_fini(&submit->exec); return ret; } +static void +v3d_submit_unlock_reservations(struct v3d_submit *submit) +{ + drm_exec_fini(&submit->exec); +} + /** * v3d_lookup_bos() - Sets up job->bo[] with the GEM objects * referenced by the job. - * @dev: DRM device * @file_priv: DRM file for this fd * @job: V3D job being set up * @bo_handles: GEM handles @@ -64,27 +84,46 @@ fail: * failure, because that will happen at `v3d_job_free()`. */ static int -v3d_lookup_bos(struct drm_device *dev, - struct drm_file *file_priv, - struct v3d_job *job, - u64 bo_handles, - u32 bo_count) +v3d_lookup_bos(struct drm_file *file_priv, struct v3d_job *job, + u64 bo_handles, u32 bo_count) { - job->bo_count = bo_count; - - if (!job->bo_count) { - /* See comment on bo_index for why we have to check - * this. - */ - drm_warn(dev, "Rendering requires BOs\n"); + if (!bo_count) { + drm_warn(&job->v3d->drm, "Rendering requires BOs\n"); return -EINVAL; } + job->bo_count = bo_count; + return drm_gem_objects_lookup(file_priv, (void __user *)(uintptr_t)bo_handles, job->bo_count, &job->bo); } +/** + * v3d_job_reference_bos() - Share another job's BOs with @dst + * @dst: job that acquires references to the BOs + * @src: job whose already-resolved BO list is shared + * + * For submissions with multiple jobs that use the same BOs, a trailing job + * shouldn't look the handles up again, as it could cause inconsistencies. + * Instead, it should reference the previous job's BOs. + */ +static int +v3d_job_reference_bos(struct v3d_job *dst, struct v3d_job *src) +{ + dst->bo = kvmalloc_objs(*dst->bo, src->bo_count); + if (!dst->bo) + return -ENOMEM; + + dst->bo_count = src->bo_count; + for (int i = 0; i < dst->bo_count; i++) { + dst->bo[i] = src->bo[i]; + drm_gem_object_get(dst->bo[i]); + } + + return 0; +} + static void v3d_job_free(struct kref *ref) { @@ -162,74 +201,88 @@ void v3d_job_put(struct v3d_job *job) } static int -v3d_job_allocate(struct v3d_dev *v3d, void **container, size_t size) +v3d_job_add_syncobjs(struct v3d_job *job, struct drm_file *file_priv, + u32 in_sync, struct v3d_submit_ext *se) { - *container = kcalloc(1, size, GFP_KERNEL); - if (!*container) { - drm_err(&v3d->drm, "Cannot allocate memory for V3D job.\n"); - return -ENOMEM; + bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); + struct v3d_dev *v3d = job->v3d; + int ret = 0; + + if (!has_multisync) { + ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, + in_sync, 0); + // TODO: Investigate why this was filtered out for the IOCTL. + if (ret && ret != -ENOENT) + return ret; + return 0; + } + + if (se->in_sync_count && se->wait_stage == job->queue) { + struct drm_v3d_sem __user *handle = u64_to_user_ptr(se->in_syncs); + + for (int i = 0; i < se->in_sync_count; i++) { + struct drm_v3d_sem in; + + if (copy_from_user(&in, handle++, sizeof(in))) { + drm_dbg(&v3d->drm, "Failed to copy wait dep handle.\n"); + return -EFAULT; + } + + ret = drm_sched_job_add_syncobj_dependency(&job->base, + file_priv, in.handle, 0); + // TODO: Investigate why this was filtered out for the IOCTL. + if (ret && ret != -ENOENT) + return ret; + } } return 0; } -static void -v3d_job_deallocate(void **container) -{ - kfree(*container); - *container = NULL; -} +static const struct { + size_t size; + void (*free)(struct kref *ref); + bool has_implicit_dep; +} v3d_job_types[] = { + [V3D_BIN] = { sizeof(struct v3d_bin_job), v3d_job_free, false }, + [V3D_RENDER] = { sizeof(struct v3d_render_job), v3d_render_job_free, true }, + [V3D_TFU] = { sizeof(struct v3d_tfu_job), v3d_job_free, true }, + [V3D_CSD] = { sizeof(struct v3d_csd_job), v3d_job_free, true }, + [V3D_CACHE_CLEAN] = { sizeof(struct v3d_job), v3d_job_free, false }, + [V3D_CPU] = { sizeof(struct v3d_cpu_job), v3d_cpu_job_free, true }, +}; -static int -v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv, - struct v3d_job *job, void (*free)(struct kref *ref), - u32 in_sync, struct v3d_submit_ext *se, enum v3d_queue queue) +static struct v3d_job * +v3d_submit_add_job(struct v3d_submit *submit, enum v3d_queue queue) { - struct v3d_file_priv *v3d_priv = file_priv->driver_priv; - bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); - int ret, i; + struct v3d_file_priv *v3d_priv = submit->file_priv->driver_priv; + struct v3d_dev *v3d = submit->v3d; + struct v3d_job *job; + int ret; + + if (queue >= V3D_MAX_QUEUES) + return ERR_PTR(-EINVAL); + + job = kzalloc(v3d_job_types[queue].size, GFP_KERNEL); + if (!job) + return ERR_PTR(-ENOMEM); job->v3d = v3d; - job->free = free; + job->queue = queue; job->file_priv = v3d_priv; + job->free = v3d_job_types[queue].free; + job->has_implicit_dep = v3d_job_types[queue].has_implicit_dep; ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue], - 1, v3d_priv, file_priv->client_id); + 1, v3d_priv, submit->file_priv->client_id); if (ret) - return ret; - - if (has_multisync) { - if (se->in_sync_count && se->wait_stage == queue) { - struct drm_v3d_sem __user *handle = u64_to_user_ptr(se->in_syncs); - - for (i = 0; i < se->in_sync_count; i++) { - struct drm_v3d_sem in; - - if (copy_from_user(&in, handle++, sizeof(in))) { - ret = -EFAULT; - drm_dbg(&v3d->drm, "Failed to copy wait dep handle.\n"); - goto fail_job_init; - } - ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in.handle, 0); - - // TODO: Investigate why this was filtered out for the IOCTL. - if (ret && ret != -ENOENT) - goto fail_job_init; - } - } - } else { - ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in_sync, 0); - - // TODO: Investigate why this was filtered out for the IOCTL. - if (ret && ret != -ENOENT) - goto fail_job_init; - } + goto fail_free; /* CPU jobs don't require hardware resources */ if (queue != V3D_CPU) { ret = v3d_pm_runtime_get(v3d); if (ret) - goto fail_job_init; + goto fail_sched_job; job->has_pm_ref = true; } @@ -238,52 +291,86 @@ v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv, job->client_stats = v3d_stats_get(v3d_priv->stats[queue]); job->global_stats = v3d_stats_get(v3d->queue[queue].stats); - return 0; + submit->jobs[submit->job_count++] = job; -fail_job_init: + return job; + +fail_sched_job: drm_sched_job_cleanup(&job->base); - return ret; +fail_free: + kfree(job); + return ERR_PTR(ret); } static void -v3d_push_job(struct v3d_job *job) +v3d_submit_put_jobs(struct v3d_submit *submit) { - drm_sched_job_arm(&job->base); + for (int i = 0; i < submit->job_count; i++) + v3d_job_put(submit->jobs[i]); +} - job->done_fence = dma_fence_get(&job->base.s_fence->finished); +static void +v3d_submit_cleanup_jobs(struct v3d_submit *submit) +{ + for (int i = 0; i < submit->job_count; i++) + v3d_job_cleanup(submit->jobs[i]); +} - /* put by scheduler job completion */ - kref_get(&job->refcount); +static int +v3d_attach_perfmon_to_jobs(struct v3d_submit *submit, u32 perfmon_id) +{ + struct v3d_file_priv *v3d_priv = submit->file_priv->driver_priv; + struct v3d_dev *v3d = submit->v3d; + struct v3d_perfmon *perfmon; - drm_sched_entity_push_job(&job->base); + if (!perfmon_id) + return 0; + + if (v3d->global_perfmon) + return -EAGAIN; + + perfmon = v3d_perfmon_find(v3d_priv, perfmon_id); + if (!perfmon) + return -ENOENT; + + for (int i = 0; i < submit->job_count; i++) { + submit->jobs[i]->perfmon = perfmon; + if (i != 0) + v3d_perfmon_get(perfmon); + } + + return 0; } static void -v3d_attach_fences_and_unlock_reservation(struct drm_file *file_priv, - struct v3d_job *job, - struct ww_acquire_ctx *acquire_ctx, - u32 out_sync, - struct v3d_submit_ext *se, - struct dma_fence *done_fence) +v3d_attach_fences_and_unlock_reservation(struct v3d_submit *submit, + u32 out_sync, struct v3d_submit_ext *se) { - struct drm_syncobj *sync_out; bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); - int i; + struct v3d_job *last_job = submit->jobs[submit->job_count - 1]; + struct drm_syncobj *sync_out; - for (i = 0; i < job->bo_count; i++) { - /* XXX: Use shared fences for read-only objects. */ - dma_resv_add_fence(job->bo[i]->resv, job->done_fence, - DMA_RESV_USAGE_WRITE); + /* The submission's last fence covers the entire submission. Attach it + * to every BO touched by any job in the submission. + */ + for (int i = 0; i < submit->job_count; i++) { + struct v3d_job *job = submit->jobs[i]; + + for (int j = 0; j < job->bo_count; j++) { + /* XXX: Use shared fences for read-only objects. */ + dma_resv_add_fence(job->bo[j]->resv, last_job->done_fence, + DMA_RESV_USAGE_WRITE); + } } - drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx); + v3d_submit_unlock_reservations(submit); /* Update the return sync object for the job */ /* If it only supports a single signal semaphore*/ if (!has_multisync) { - sync_out = drm_syncobj_find(file_priv, out_sync); + sync_out = drm_syncobj_find(submit->file_priv, out_sync); if (sync_out) { - drm_syncobj_replace_fence(sync_out, done_fence); + drm_syncobj_replace_fence(sync_out, last_job->done_fence); drm_syncobj_put(sync_out); } return; @@ -291,56 +378,91 @@ v3d_attach_fences_and_unlock_reservation(struct drm_file *file_priv, /* If multiple semaphores extension is supported */ if (se->out_sync_count) { - for (i = 0; i < se->out_sync_count; i++) { + for (int i = 0; i < se->out_sync_count; i++) { drm_syncobj_replace_fence(se->out_syncs[i].syncobj, - done_fence); + last_job->done_fence); drm_syncobj_put(se->out_syncs[i].syncobj); } kvfree(se->out_syncs); } } +static void +v3d_push_job(struct v3d_job *job) +{ + drm_sched_job_arm(&job->base); + + job->done_fence = dma_fence_get(&job->base.s_fence->finished); + + /* put by scheduler job completion */ + kref_get(&job->refcount); + + drm_sched_entity_push_job(&job->base); +} + static int -v3d_setup_csd_jobs_and_bos(struct drm_file *file_priv, - struct v3d_dev *v3d, - struct drm_v3d_submit_csd *args, - struct v3d_csd_job **job, - struct v3d_job **clean_job, - struct v3d_submit_ext *se, - struct ww_acquire_ctx *acquire_ctx) +v3d_submit_jobs(struct v3d_submit *submit, u32 out_sync, + struct v3d_submit_ext *se) { - int ret; + struct v3d_dev *v3d = submit->v3d; + int ret = 0; - ret = v3d_job_allocate(v3d, (void *)job, sizeof(**job)); - if (ret) - return ret; + mutex_lock(&v3d->sched_lock); - ret = v3d_job_init(v3d, file_priv, &(*job)->base, - v3d_job_free, args->in_sync, se, V3D_CSD); - if (ret) { - v3d_job_deallocate((void *)job); - return ret; + for (int i = 0; i < submit->job_count; i++) { + struct v3d_job *job = submit->jobs[i]; + + v3d_push_job(job); + + if (i + 1 < submit->job_count) { + ret = drm_sched_job_add_dependency(&submit->jobs[i + 1]->base, + dma_fence_get(job->done_fence)); + if (ret) + goto err; + } } - ret = v3d_job_allocate(v3d, (void *)clean_job, sizeof(**clean_job)); + mutex_unlock(&v3d->sched_lock); + + v3d_attach_fences_and_unlock_reservation(submit, out_sync, se); + v3d_submit_put_jobs(submit); + + return 0; + +err: + mutex_unlock(&v3d->sched_lock); + return ret; +} + +static int +v3d_setup_csd_jobs_and_bos(struct v3d_submit *submit, + struct drm_v3d_submit_csd *args, + struct v3d_submit_ext *se) +{ + struct v3d_csd_job *job; + struct v3d_job *clean_job; + int ret; + + job = (struct v3d_csd_job *)v3d_submit_add_job(submit, V3D_CSD); + if (IS_ERR(job)) + return PTR_ERR(job); + + ret = v3d_job_add_syncobjs(&job->base, submit->file_priv, args->in_sync, se); if (ret) return ret; - ret = v3d_job_init(v3d, file_priv, *clean_job, - v3d_job_free, 0, NULL, V3D_CACHE_CLEAN); - if (ret) { - v3d_job_deallocate((void *)clean_job); + ret = v3d_lookup_bos(submit->file_priv, &job->base, args->bo_handles, + args->bo_handle_count); + if (ret) return ret; - } - (*job)->args = *args; + job->args = *args; - ret = v3d_lookup_bos(&v3d->drm, file_priv, *clean_job, - args->bo_handles, args->bo_handle_count); - if (ret) - return ret; + clean_job = v3d_submit_add_job(submit, V3D_CACHE_CLEAN); + if (IS_ERR(clean_job)) + return PTR_ERR(clean_job); - return v3d_lock_bo_reservations(*clean_job, acquire_ctx); + return v3d_job_reference_bos(clean_job, &job->base); } static void @@ -489,6 +611,7 @@ v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv, } job->job_type = V3D_CPU_JOB_TYPE_INDIRECT_CSD; + info->args = indirect_csd.submit; info->offset = indirect_csd.offset; info->wg_size = indirect_csd.wg_size; memcpy(&info->wg_uniform_offsets, &indirect_csd.wg_uniform_offsets, @@ -498,9 +621,7 @@ v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv, if (!info->indirect) return -ENOENT; - return v3d_setup_csd_jobs_and_bos(file_priv, v3d, &indirect_csd.submit, - &info->job, &info->clean_job, - NULL, &info->acquire_ctx); + return 0; } /* Get data for the query timestamp job submission. */ @@ -907,18 +1028,15 @@ int v3d_submit_cl_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct v3d_dev *v3d = to_v3d_dev(dev); - struct v3d_file_priv *v3d_priv = file_priv->driver_priv; + struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; struct drm_v3d_submit_cl *args = data; struct v3d_submit_ext se = {0}; struct v3d_bin_job *bin = NULL; - struct v3d_render_job *render = NULL; - struct v3d_job *clean_job = NULL; - struct v3d_job *last_job; - struct ww_acquire_ctx acquire_ctx; - int ret = 0; + struct v3d_render_job *render; + struct v3d_job *clean_job; + int ret; - trace_v3d_submit_cl_ioctl(&v3d->drm, args->rcl_start, args->rcl_end); + trace_v3d_submit_cl_ioctl(dev, args->rcl_start, args->rcl_end); if (args->pad) return -EINVAL; @@ -938,30 +1056,10 @@ v3d_submit_cl_ioctl(struct drm_device *dev, void *data, } } - ret = v3d_job_allocate(v3d, (void *)&render, sizeof(*render)); - if (ret) - return ret; - - ret = v3d_job_init(v3d, file_priv, &render->base, - v3d_render_job_free, args->in_sync_rcl, &se, V3D_RENDER); - if (ret) { - v3d_job_deallocate((void *)&render); - goto fail; - } - - render->start = args->rcl_start; - render->end = args->rcl_end; - INIT_LIST_HEAD(&render->unref_list); - if (args->bcl_start != args->bcl_end) { - ret = v3d_job_allocate(v3d, (void *)&bin, sizeof(*bin)); - if (ret) - goto fail; - - ret = v3d_job_init(v3d, file_priv, &bin->base, - v3d_job_free, args->in_sync_bcl, &se, V3D_BIN); - if (ret) { - v3d_job_deallocate((void *)&bin); + bin = (struct v3d_bin_job *)v3d_submit_add_job(&submit, V3D_BIN); + if (IS_ERR(bin)) { + ret = PTR_ERR(bin); goto fail; } @@ -970,100 +1068,71 @@ v3d_submit_cl_ioctl(struct drm_device *dev, void *data, bin->qma = args->qma; bin->qms = args->qms; bin->qts = args->qts; - bin->render = render; - } - if (args->flags & DRM_V3D_SUBMIT_CL_FLUSH_CACHE) { - ret = v3d_job_allocate(v3d, (void *)&clean_job, sizeof(*clean_job)); + ret = v3d_job_add_syncobjs(&bin->base, file_priv, args->in_sync_bcl, + &se); if (ret) goto fail; + } - ret = v3d_job_init(v3d, file_priv, clean_job, - v3d_job_free, 0, NULL, V3D_CACHE_CLEAN); - if (ret) { - v3d_job_deallocate((void *)&clean_job); - goto fail; - } - - last_job = clean_job; - } else { - last_job = &render->base; + render = (struct v3d_render_job *)v3d_submit_add_job(&submit, V3D_RENDER); + if (IS_ERR(render)) { + ret = PTR_ERR(render); + goto fail; } - ret = v3d_lookup_bos(dev, file_priv, last_job, - args->bo_handles, args->bo_handle_count); + INIT_LIST_HEAD(&render->unref_list); + render->start = args->rcl_start; + render->end = args->rcl_end; + + if (bin) + bin->render = render; + + ret = v3d_job_add_syncobjs(&render->base, file_priv, args->in_sync_rcl, &se); if (ret) goto fail; - ret = v3d_lock_bo_reservations(last_job, &acquire_ctx); + /* + * We don't associate the BOs with the BIN job. Fences are only + * attached to the last job in the submission chain, and BIN jobs + * don't need implicit dependencies since depending on results from + * another context is not a realistic scenario for binning. + */ + ret = v3d_lookup_bos(submit.file_priv, &render->base, + args->bo_handles, args->bo_handle_count); if (ret) goto fail; - if (args->perfmon_id) { - if (v3d->global_perfmon) { - ret = -EAGAIN; - goto fail_perfmon; - } - - render->base.perfmon = v3d_perfmon_find(v3d_priv, - args->perfmon_id); - - if (!render->base.perfmon) { - ret = -ENOENT; - goto fail_perfmon; + if (args->flags & DRM_V3D_SUBMIT_CL_FLUSH_CACHE) { + clean_job = v3d_submit_add_job(&submit, V3D_CACHE_CLEAN); + if (IS_ERR(clean_job)) { + ret = PTR_ERR(clean_job); + goto fail; } - } - mutex_lock(&v3d->sched_lock); - if (bin) { - bin->base.perfmon = render->base.perfmon; - v3d_perfmon_get(bin->base.perfmon); - v3d_push_job(&bin->base); - - ret = drm_sched_job_add_dependency(&render->base.base, - dma_fence_get(bin->base.done_fence)); + ret = v3d_job_reference_bos(clean_job, &render->base); if (ret) - goto fail_unreserve; - } - - v3d_push_job(&render->base); - - if (clean_job) { - struct dma_fence *render_fence = - dma_fence_get(render->base.done_fence); - ret = drm_sched_job_add_dependency(&clean_job->base, - render_fence); - if (ret) - goto fail_unreserve; - clean_job->perfmon = render->base.perfmon; - v3d_perfmon_get(clean_job->perfmon); - v3d_push_job(clean_job); + goto fail; } - mutex_unlock(&v3d->sched_lock); + ret = v3d_attach_perfmon_to_jobs(&submit, args->perfmon_id); + if (ret) + goto fail; - v3d_attach_fences_and_unlock_reservation(file_priv, - last_job, - &acquire_ctx, - args->out_sync, - &se, - last_job->done_fence); + ret = v3d_submit_lock_reservations(&submit); + if (ret) + goto fail; - v3d_job_put(&bin->base); - v3d_job_put(&render->base); - v3d_job_put(clean_job); + ret = v3d_submit_jobs(&submit, args->out_sync, &se); + if (ret) + goto fail_unreserve; return 0; fail_unreserve: - mutex_unlock(&v3d->sched_lock); -fail_perfmon: - drm_gem_unlock_reservations(last_job->bo, - last_job->bo_count, &acquire_ctx); + v3d_submit_unlock_reservations(&submit); fail: - v3d_job_cleanup((void *)bin); - v3d_job_cleanup((void *)render); - v3d_job_cleanup(clean_job); + v3d_submit_cleanup_jobs(&submit); v3d_put_multisync_post_deps(&se); return ret; @@ -1082,14 +1151,13 @@ int v3d_submit_tfu_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct v3d_dev *v3d = to_v3d_dev(dev); + struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; struct drm_v3d_submit_tfu *args = data; struct v3d_submit_ext se = {0}; - struct v3d_tfu_job *job = NULL; - struct ww_acquire_ctx acquire_ctx; + struct v3d_tfu_job *job; int ret = 0; - trace_v3d_submit_tfu_ioctl(&v3d->drm, args->iia); + trace_v3d_submit_tfu_ioctl(dev, args->iia); if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { drm_dbg(dev, "invalid flags: %d\n", args->flags); @@ -1104,17 +1172,16 @@ v3d_submit_tfu_ioctl(struct drm_device *dev, void *data, } } - ret = v3d_job_allocate(v3d, (void *)&job, sizeof(*job)); - if (ret) - return ret; - - ret = v3d_job_init(v3d, file_priv, &job->base, - v3d_job_free, args->in_sync, &se, V3D_TFU); - if (ret) { - v3d_job_deallocate((void *)&job); + job = (struct v3d_tfu_job *)v3d_submit_add_job(&submit, V3D_TFU); + if (IS_ERR(job)) { + ret = PTR_ERR(job); goto fail; } + ret = v3d_job_add_syncobjs(&job->base, file_priv, args->in_sync, &se); + if (ret) + goto fail; + job->base.bo = kzalloc_objs(*job->base.bo, ARRAY_SIZE(args->bo_handles)); if (!job->base.bo) { ret = -ENOMEM; @@ -1142,26 +1209,20 @@ v3d_submit_tfu_ioctl(struct drm_device *dev, void *data, job->base.bo[job->base.bo_count] = bo; } - ret = v3d_lock_bo_reservations(&job->base, &acquire_ctx); + ret = v3d_submit_lock_reservations(&submit); if (ret) goto fail; - mutex_lock(&v3d->sched_lock); - v3d_push_job(&job->base); - mutex_unlock(&v3d->sched_lock); - - v3d_attach_fences_and_unlock_reservation(file_priv, - &job->base, &acquire_ctx, - args->out_sync, - &se, - job->base.done_fence); - - v3d_job_put(&job->base); + ret = v3d_submit_jobs(&submit, args->out_sync, &se); + if (ret) + goto fail_unreserve; return 0; +fail_unreserve: + v3d_submit_unlock_reservations(&submit); fail: - v3d_job_cleanup((void *)job); + v3d_submit_cleanup_jobs(&submit); v3d_put_multisync_post_deps(&se); return ret; @@ -1180,21 +1241,17 @@ int v3d_submit_csd_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct v3d_dev *v3d = to_v3d_dev(dev); - struct v3d_file_priv *v3d_priv = file_priv->driver_priv; + struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; struct drm_v3d_submit_csd *args = data; struct v3d_submit_ext se = {0}; - struct v3d_csd_job *job = NULL; - struct v3d_job *clean_job = NULL; - struct ww_acquire_ctx acquire_ctx; int ret; - trace_v3d_submit_csd_ioctl(&v3d->drm, args->cfg[5], args->cfg[6]); + trace_v3d_submit_csd_ioctl(dev, args->cfg[5], args->cfg[6]); if (args->pad) return -EINVAL; - if (!v3d_has_csd(v3d)) { + if (!v3d_has_csd(submit.v3d)) { drm_warn(dev, "Attempting CSD submit on non-CSD hardware\n"); return -EINVAL; } @@ -1212,57 +1269,28 @@ v3d_submit_csd_ioctl(struct drm_device *dev, void *data, } } - ret = v3d_setup_csd_jobs_and_bos(file_priv, v3d, args, - &job, &clean_job, &se, - &acquire_ctx); + ret = v3d_setup_csd_jobs_and_bos(&submit, args, &se); if (ret) goto fail; - if (args->perfmon_id) { - if (v3d->global_perfmon) { - ret = -EAGAIN; - goto fail_perfmon; - } - - job->base.perfmon = v3d_perfmon_find(v3d_priv, - args->perfmon_id); - if (!job->base.perfmon) { - ret = -ENOENT; - goto fail_perfmon; - } - } + ret = v3d_attach_perfmon_to_jobs(&submit, args->perfmon_id); + if (ret) + goto fail; - mutex_lock(&v3d->sched_lock); - v3d_push_job(&job->base); + ret = v3d_submit_lock_reservations(&submit); + if (ret) + goto fail; - ret = drm_sched_job_add_dependency(&clean_job->base, - dma_fence_get(job->base.done_fence)); + ret = v3d_submit_jobs(&submit, args->out_sync, &se); if (ret) goto fail_unreserve; - v3d_push_job(clean_job); - mutex_unlock(&v3d->sched_lock); - - v3d_attach_fences_and_unlock_reservation(file_priv, - clean_job, - &acquire_ctx, - args->out_sync, - &se, - clean_job->done_fence); - - v3d_job_put(&job->base); - v3d_job_put(clean_job); - return 0; fail_unreserve: - mutex_unlock(&v3d->sched_lock); -fail_perfmon: - drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count, - &acquire_ctx); + v3d_submit_unlock_reservations(&submit); fail: - v3d_job_cleanup((void *)job); - v3d_job_cleanup(clean_job); + v3d_submit_cleanup_jobs(&submit); v3d_put_multisync_post_deps(&se); return ret; @@ -1291,13 +1319,14 @@ v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct v3d_dev *v3d = to_v3d_dev(dev); + struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; + struct v3d_submit indirect_submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; struct drm_v3d_submit_cpu *args = data; struct v3d_submit_ext se = {0}; struct v3d_submit_ext *out_se = NULL; struct v3d_cpu_job *cpu_job = NULL; struct v3d_csd_job *csd_job = NULL; struct v3d_job *clean_job = NULL; - struct ww_acquire_ctx acquire_ctx; int ret; if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { @@ -1305,9 +1334,9 @@ v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, return -EINVAL; } - ret = v3d_job_allocate(v3d, (void *)&cpu_job, sizeof(*cpu_job)); - if (ret) - return ret; + cpu_job = (struct v3d_cpu_job *)v3d_submit_add_job(&submit, V3D_CPU); + if (IS_ERR(cpu_job)) + return PTR_ERR(cpu_job); if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { ret = v3d_get_extensions(file_priv, args->extensions, &se, cpu_job); @@ -1332,23 +1361,36 @@ v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, trace_v3d_submit_cpu_ioctl(&v3d->drm, cpu_job->job_type); - ret = v3d_job_init(v3d, file_priv, &cpu_job->base, - v3d_cpu_job_free, 0, &se, V3D_CPU); - if (ret) { - v3d_job_deallocate((void *)&cpu_job); + ret = v3d_job_add_syncobjs(&cpu_job->base, file_priv, 0, &se); + if (ret) goto fail; - } - clean_job = cpu_job->indirect_csd.clean_job; - csd_job = cpu_job->indirect_csd.job; + if (cpu_job->job_type == V3D_CPU_JOB_TYPE_INDIRECT_CSD) { + ret = v3d_setup_csd_jobs_and_bos(&indirect_submit, + &cpu_job->indirect_csd.args, + NULL); + if (ret) + goto fail; + + ret = v3d_submit_lock_reservations(&indirect_submit); + if (ret) + goto fail; + + cpu_job->indirect_csd.job = container_of(indirect_submit.jobs[0], + struct v3d_csd_job, base); + cpu_job->indirect_csd.clean_job = indirect_submit.jobs[1]; + + clean_job = cpu_job->indirect_csd.clean_job; + csd_job = cpu_job->indirect_csd.job; + } if (args->bo_handle_count) { - ret = v3d_lookup_bos(dev, file_priv, &cpu_job->base, + ret = v3d_lookup_bos(submit.file_priv, &cpu_job->base, args->bo_handles, args->bo_handle_count); if (ret) goto fail; - ret = v3d_lock_bo_reservations(&cpu_job->base, &acquire_ctx); + ret = v3d_submit_lock_reservations(&submit); if (ret) goto fail; } @@ -1380,41 +1422,28 @@ v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, out_se = (cpu_job->job_type == V3D_CPU_JOB_TYPE_INDIRECT_CSD) ? NULL : &se; - v3d_attach_fences_and_unlock_reservation(file_priv, - &cpu_job->base, - &acquire_ctx, 0, - out_se, cpu_job->base.done_fence); + v3d_attach_fences_and_unlock_reservation(&submit, 0, out_se); switch (cpu_job->job_type) { case V3D_CPU_JOB_TYPE_INDIRECT_CSD: - v3d_attach_fences_and_unlock_reservation(file_priv, - clean_job, - &cpu_job->indirect_csd.acquire_ctx, - 0, &se, clean_job->done_fence); + v3d_attach_fences_and_unlock_reservation(&indirect_submit, 0, &se); break; default: break; } - v3d_job_put(&cpu_job->base); - v3d_job_put(&csd_job->base); - v3d_job_put(clean_job); + v3d_submit_put_jobs(&submit); + v3d_submit_put_jobs(&indirect_submit); return 0; fail_unreserve: mutex_unlock(&v3d->sched_lock); - - drm_gem_unlock_reservations(cpu_job->base.bo, cpu_job->base.bo_count, - &acquire_ctx); - - drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count, - &cpu_job->indirect_csd.acquire_ctx); - + v3d_submit_unlock_reservations(&submit); + v3d_submit_unlock_reservations(&indirect_submit); fail: - v3d_job_cleanup((void *)cpu_job); - v3d_job_cleanup((void *)csd_job); - v3d_job_cleanup(clean_job); + v3d_submit_cleanup_jobs(&submit); + v3d_submit_cleanup_jobs(&indirect_submit); v3d_put_multisync_post_deps(&se); return ret; diff --git a/drivers/gpu/drm/xe/tests/xe_migrate.c b/drivers/gpu/drm/xe/tests/xe_migrate.c index 3c1be809be82d8..f10d9513747b48 100644 --- a/drivers/gpu/drm/xe/tests/xe_migrate.c +++ b/drivers/gpu/drm/xe/tests/xe_migrate.c @@ -198,8 +198,7 @@ static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test, err = xe_bo_vmap(bo); if (err) { - KUNIT_FAIL(test, "Failed to vmap our pagetables: %li\n", - PTR_ERR(bo)); + KUNIT_FAIL(test, "Failed to vmap our pagetables: %d\n", err); return; } diff --git a/drivers/gpu/drm/xe/xe_gt_idle.c b/drivers/gpu/drm/xe/xe_gt_idle.c index 04b24e1c8b78e9..7dc9873aef545d 100644 --- a/drivers/gpu/drm/xe/xe_gt_idle.c +++ b/drivers/gpu/drm/xe/xe_gt_idle.c @@ -248,7 +248,8 @@ int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p) pg_status = xe_mmio_read32(>->mmio, POWERGATE_DOMAIN_STATUS); } - if (gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK) { + if (gt->info.engine_mask & + (XE_HW_ENGINE_RCS_MASK | XE_HW_ENGINE_CCS_MASK)) { drm_printf(p, "Render Power Gating Enabled: %s\n", str_yes_no(pg_enabled & RENDER_POWERGATE_ENABLE)); diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 1c92b96e4c3246..eb426e85858878 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -1537,8 +1537,14 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) if (!skip_timeout_check && !check_timeout(q, job)) goto rearm; + /* + * Killed queues must not newly wedge the device, but preserve an + * already-wedged state to avoid warning on teardown timeouts. + */ if (!exec_queue_killed(q)) wedged = guc_submit_hint_wedged(exec_queue_to_guc(q)); + else + wedged = xe_device_wedged(xe); set_exec_queue_banned(q); diff --git a/drivers/gpu/drm/xe/xe_pagefault.c b/drivers/gpu/drm/xe/xe_pagefault.c index dd3c068e1a3991..dbf8f71d3328a7 100644 --- a/drivers/gpu/drm/xe/xe_pagefault.c +++ b/drivers/gpu/drm/xe/xe_pagefault.c @@ -158,7 +158,7 @@ static struct xe_vm *xe_pagefault_asid_to_vm(struct xe_device *xe, u32 asid) down_read(&xe->usm.lock); vm = xa_load(&xe->usm.asid_to_vm, asid); - if (vm && (xe_vm_in_fault_mode(vm) || xe_vm_has_scratch(vm))) + if (vm && xe_vm_in_fault_mode(vm)) xe_vm_get(vm); else vm = ERR_PTR(-EINVAL); diff --git a/drivers/gpu/drm/xe/xe_ras.c b/drivers/gpu/drm/xe/xe_ras.c index 4cb16b419b0c66..96702234d7ece2 100644 --- a/drivers/gpu/drm/xe/xe_ras.c +++ b/drivers/gpu/drm/xe/xe_ras.c @@ -4,11 +4,14 @@ */ #include "xe_device.h" +#include "xe_pm.h" #include "xe_printk.h" #include "xe_ras.h" #include "xe_ras_types.h" #include "xe_sysctrl.h" #include "xe_sysctrl_event_types.h" +#include "xe_sysctrl_mailbox.h" +#include "xe_sysctrl_mailbox_types.h" /* Severity of detected errors */ enum xe_ras_severity { @@ -50,6 +53,36 @@ static const char *const xe_ras_components[] = { }; static_assert(ARRAY_SIZE(xe_ras_components) == XE_RAS_COMP_MAX); +static u8 drm_to_xe_ras_severity(u8 severity) +{ + switch (severity) { + case DRM_XE_RAS_ERR_SEV_CORRECTABLE: + return XE_RAS_SEV_CORRECTABLE; + case DRM_XE_RAS_ERR_SEV_UNCORRECTABLE: + return XE_RAS_SEV_UNCORRECTABLE; + default: + return XE_RAS_SEV_NOT_SUPPORTED; + } +} + +static u8 drm_to_xe_ras_component(u8 component) +{ + switch (component) { + case DRM_XE_RAS_ERR_COMP_CORE_COMPUTE: + return XE_RAS_COMP_CORE_COMPUTE; + case DRM_XE_RAS_ERR_COMP_SOC_INTERNAL: + return XE_RAS_COMP_SOC_INTERNAL; + case DRM_XE_RAS_ERR_COMP_DEVICE_MEMORY: + return XE_RAS_COMP_DEVICE_MEMORY; + case DRM_XE_RAS_ERR_COMP_PCIE: + return XE_RAS_COMP_PCIE; + case DRM_XE_RAS_ERR_COMP_FABRIC: + return XE_RAS_COMP_FABRIC; + default: + return XE_RAS_COMP_NOT_SUPPORTED; + } +} + static inline const char *sev_to_str(u8 severity) { if (severity >= XE_RAS_SEV_MAX) @@ -91,3 +124,61 @@ void xe_ras_counter_threshold_crossed(struct xe_device *xe, comp_to_str(component), sev_to_str(severity)); } } + +static int get_counter(struct xe_device *xe, struct xe_ras_error_class *counter, u32 *value) +{ + struct xe_ras_get_counter_response response = {0}; + struct xe_ras_get_counter_request request = {0}; + struct xe_sysctrl_mailbox_command command = {0}; + struct xe_ras_error_common *common; + size_t rlen; + int ret; + + request.counter = *counter; + + xe_sysctrl_create_command(&command, XE_SYSCTRL_GROUP_GFSP, XE_SYSCTRL_CMD_GET_COUNTER, + &request, sizeof(request), &response, sizeof(response)); + + ret = xe_sysctrl_send_command(&xe->sc, &command, &rlen); + if (ret) { + xe_err(xe, "sysctrl: failed to get counter %d\n", ret); + return ret; + } + + if (rlen != sizeof(response)) { + xe_err(xe, "sysctrl: unexpected get counter response length %zu (expected %zu)\n", + rlen, sizeof(response)); + return -EIO; + } + + common = &response.counter.common; + *value = response.value; + + xe_dbg(xe, "[RAS]: get counter %u for %s %s\n", *value, comp_to_str(common->component), + sev_to_str(common->severity)); + + return 0; +} + +/** + * xe_ras_get_counter() - Get error counter value + * @xe: Xe device instance + * @severity: Error severity to be queried (&enum drm_xe_ras_error_severity) + * @component: Error component to be queried (&enum drm_xe_ras_error_component) + * @value: Counter value + * + * This function retrieves the value of a specific error counter based on + * the error severity and component. + * + * Return: 0 on success, negative error code on failure. + */ +int xe_ras_get_counter(struct xe_device *xe, u8 severity, u8 component, u32 *value) +{ + struct xe_ras_error_class counter = {0}; + + counter.common.severity = drm_to_xe_ras_severity(severity); + counter.common.component = drm_to_xe_ras_component(component); + + guard(xe_pm_runtime)(xe); + return get_counter(xe, &counter, value); +} diff --git a/drivers/gpu/drm/xe/xe_ras.h b/drivers/gpu/drm/xe/xe_ras.h index ea90593b62dc12..e148debd5d418d 100644 --- a/drivers/gpu/drm/xe/xe_ras.h +++ b/drivers/gpu/drm/xe/xe_ras.h @@ -6,10 +6,13 @@ #ifndef _XE_RAS_H_ #define _XE_RAS_H_ +#include + struct xe_device; struct xe_sysctrl_event_response; void xe_ras_counter_threshold_crossed(struct xe_device *xe, struct xe_sysctrl_event_response *response); +int xe_ras_get_counter(struct xe_device *xe, u8 severity, u8 component, u32 *value); #endif diff --git a/drivers/gpu/drm/xe/xe_ras_types.h b/drivers/gpu/drm/xe/xe_ras_types.h index 4e63c67f806a26..fdfebaeb5ed261 100644 --- a/drivers/gpu/drm/xe/xe_ras_types.h +++ b/drivers/gpu/drm/xe/xe_ras_types.h @@ -70,4 +70,30 @@ struct xe_ras_threshold_crossed { struct xe_ras_error_class counters[XE_RAS_NUM_COUNTERS]; } __packed; +/** + * struct xe_ras_get_counter_request - Request structure for get counter + */ +struct xe_ras_get_counter_request { + /** @counter: Error counter to be queried */ + struct xe_ras_error_class counter; + /** @reserved: Reserved for future use */ + u32 reserved; +} __packed; + +/** + * struct xe_ras_get_counter_response - Response structure for get counter + */ +struct xe_ras_get_counter_response { + /** @counter: Error counter that was queried */ + struct xe_ras_error_class counter; + /** @value: Current counter value */ + u32 value; + /** @timestamp: Timestamp when counter was last updated */ + u64 timestamp; + /** @threshold: Threshold value for the counter */ + u32 threshold; + /** @reserved: Reserved */ + u32 reserved[57]; +} __packed; + #endif diff --git a/drivers/gpu/drm/xe/xe_sysctrl.c b/drivers/gpu/drm/xe/xe_sysctrl.c index 1db20be8158b56..62ccc9be71b45a 100644 --- a/drivers/gpu/drm/xe/xe_sysctrl.c +++ b/drivers/gpu/drm/xe/xe_sysctrl.c @@ -85,7 +85,6 @@ int xe_sysctrl_init(struct xe_device *xe) return ret; xe->soc_remapper.set_sysctrl_region(xe, SYSCTRL_MAILBOX_INDEX); - xe_sysctrl_mailbox_init(sc); INIT_WORK(&sc->work, xe_sysctrl_work); return devm_add_action_or_reset(xe->drm.dev, sysctrl_fini, xe); @@ -114,12 +113,10 @@ void xe_sysctrl_irq_handler(struct xe_device *xe, u32 master_ctl) * @xe: xe device instance * * Invoked during system resume (S3/S4 to S0) and runtime resume from D3cold. - * Restores SoC remapper configuration and reinitializes mailbox interface. + * Restores SoC remapper configuration. */ void xe_sysctrl_pm_resume(struct xe_device *xe) { - struct xe_sysctrl *sc = &xe->sc; - if (!xe->info.has_soc_remapper_sysctrl) return; @@ -127,6 +124,4 @@ void xe_sysctrl_pm_resume(struct xe_device *xe) return; xe->soc_remapper.set_sysctrl_region(xe, SYSCTRL_MAILBOX_INDEX); - - xe_sysctrl_mailbox_init(sc); } diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c index 3caa9f15875f56..72baf1aa4b3ad6 100644 --- a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c +++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c @@ -145,6 +145,7 @@ static int sysctrl_send_frames(struct xe_sysctrl *sc, struct xe_device *xe = sc_to_xe(sc); u32 ctrl_reg, total_frames, frame; size_t bytes_sent, frame_size; + bool phase; total_frames = DIV_ROUND_UP(cmd_size, XE_SYSCTRL_MB_FRAME_SIZE); @@ -153,7 +154,8 @@ static int sysctrl_send_frames(struct xe_sysctrl *sc, return -EBUSY; } - sc->phase_bit ^= 1; + ctrl_reg = xe_mmio_read32(sc->mmio, SYSCTRL_MB_CTRL); + phase = !(ctrl_reg & SYSCTRL_FRAME_PHASE); bytes_sent = 0; for (frame = 0; frame < total_frames; frame++) { @@ -161,7 +163,6 @@ static int sysctrl_send_frames(struct xe_sysctrl *sc, if (sysctrl_write_frame(sc, mbox_cmd + bytes_sent, frame_size)) { xe_err(xe, "sysctrl: Failed to write frame %u\n", frame); - sc->phase_bit = 0; return -EIO; } @@ -169,13 +170,12 @@ static int sysctrl_send_frames(struct xe_sysctrl *sc, REG_FIELD_PREP(SYSCTRL_FRAME_CURRENT_MASK, frame) | REG_FIELD_PREP(SYSCTRL_FRAME_TOTAL_MASK, total_frames - 1) | SYSCTRL_MB_CTRL_CMD | - (sc->phase_bit ? SYSCTRL_FRAME_PHASE : 0); + (phase ? SYSCTRL_FRAME_PHASE : 0); xe_mmio_write32(sc->mmio, SYSCTRL_MB_CTRL, ctrl_reg); if (!sysctrl_wait_bit_clear(sc, SYSCTRL_MB_CTRL_RUN_BUSY, timeout_ms)) { xe_err(xe, "sysctrl: Frame %u acknowledgment timeout\n", frame); - sc->phase_bit = 0; return -ETIMEDOUT; } @@ -294,17 +294,31 @@ static int sysctrl_send_command(struct xe_sysctrl *sc, } /** - * xe_sysctrl_mailbox_init - Initialize System Controller mailbox interface - * @sc: System controller structure + * xe_sysctrl_create_command() - Create system controller command + * @command: Sysctrl command structure + * @group_id: Command group ID + * @cmd_id: Command ID + * @request: Pointer to request buffer (can be NULL) + * @request_len: Size of request buffer + * @response: Pointer to response buffer + * @response_len: Size of response buffer * - * Initialize system controller mailbox interface for communication. + * Helper function to create sysctrl command to be sent via %xe_sysctrl_send_command() */ -void xe_sysctrl_mailbox_init(struct xe_sysctrl *sc) +void xe_sysctrl_create_command(struct xe_sysctrl_mailbox_command *command, u8 group_id, u8 cmd_id, + void *request, size_t request_len, void *response, + size_t response_len) { - u32 ctrl_reg; + struct xe_sysctrl_app_msg_hdr header = {0}; - ctrl_reg = xe_mmio_read32(sc->mmio, SYSCTRL_MB_CTRL); - sc->phase_bit = (ctrl_reg & SYSCTRL_FRAME_PHASE) ? 1 : 0; + header.data = FIELD_PREP(APP_HDR_GROUP_ID_MASK, group_id) | + FIELD_PREP(APP_HDR_COMMAND_MASK, cmd_id); + + command->header = header; + command->data_in = request; + command->data_in_len = request_len; + command->data_out = response; + command->data_out_len = response_len; } /** diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h index f67e9234de482f..f0d5e3d7f5e31c 100644 --- a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h +++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h @@ -23,7 +23,9 @@ struct xe_sysctrl_mailbox_command; #define XE_SYSCTRL_APP_HDR_VERSION(hdr) \ FIELD_GET(APP_HDR_VERSION_MASK, (hdr)->data) -void xe_sysctrl_mailbox_init(struct xe_sysctrl *sc); +void xe_sysctrl_create_command(struct xe_sysctrl_mailbox_command *command, u8 group_id, u8 cmd_id, + void *request, size_t request_len, void *response, + size_t response_len); int xe_sysctrl_send_command(struct xe_sysctrl *sc, struct xe_sysctrl_mailbox_command *cmd, size_t *rdata_len); diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h index 84d7c647e7432c..b315847cbf64f8 100644 --- a/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h +++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h @@ -22,9 +22,11 @@ enum xe_sysctrl_group { /** * enum xe_sysctrl_gfsp_cmd - Commands supported by GFSP group * + * @XE_SYSCTRL_CMD_GET_COUNTER: Get error counter value * @XE_SYSCTRL_CMD_GET_PENDING_EVENT: Retrieve pending event */ enum xe_sysctrl_gfsp_cmd { + XE_SYSCTRL_CMD_GET_COUNTER = 0x03, XE_SYSCTRL_CMD_GET_PENDING_EVENT = 0x07, }; diff --git a/drivers/gpu/drm/xe/xe_sysctrl_types.h b/drivers/gpu/drm/xe/xe_sysctrl_types.h index 66ba24f43017d5..98c2f473f7c6a1 100644 --- a/drivers/gpu/drm/xe/xe_sysctrl_types.h +++ b/drivers/gpu/drm/xe/xe_sysctrl_types.h @@ -26,9 +26,6 @@ struct xe_sysctrl { /** @cmd_lock: Mutex protecting mailbox command operations */ struct mutex cmd_lock; - /** @phase_bit: Message boundary phase toggle bit (0 or 1) */ - bool phase_bit; - /** @work: Pending events worker */ struct work_struct work; diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules index f8a185103b8057..132a41859ab791 100644 --- a/drivers/gpu/drm/xe/xe_wa_oob.rules +++ b/drivers/gpu/drm/xe/xe_wa_oob.rules @@ -63,5 +63,5 @@ 16026007364 MEDIA_VERSION(3000) 14020316580 MEDIA_VERSION(1301) -14025883347 MEDIA_VERSION_RANGE(1301, 3503) +14025883347 MEDIA_VERSION_RANGE(1301, 3500) GRAPHICS_VERSION_RANGE(2004, 3005) diff --git a/drivers/gpu/host1x/bus.c b/drivers/gpu/host1x/bus.c index ea3b584990c9a6..733f53e69eacc2 100644 --- a/drivers/gpu/host1x/bus.c +++ b/drivers/gpu/host1x/bus.c @@ -508,7 +508,7 @@ static int host1x_device_add(struct host1x *host1x, * Add device even if there are no subdevs to ensure syncpoint functionality * is available regardless of whether any engine subdevices are present */ - if (list_empty(&device->subdevs)) { + if (list_empty(&device->subdevs) && !device->registered) { err = device_add(&device->dev); if (err < 0) dev_err(&device->dev, "failed to add device: %d\n", err); diff --git a/drivers/gpu/host1x/debug.c b/drivers/gpu/host1x/debug.c index 6433c00d5d7e07..b828f773fc0654 100644 --- a/drivers/gpu/host1x/debug.c +++ b/drivers/gpu/host1x/debug.c @@ -31,7 +31,7 @@ void host1x_debug_output(struct output *o, const char *fmt, ...) int len; va_start(args, fmt); - len = vsnprintf(o->buf, sizeof(o->buf), fmt, args); + len = vscnprintf(o->buf, sizeof(o->buf), fmt, args); va_end(args); o->fn(o->ctx, o->buf, len, false); @@ -43,7 +43,7 @@ void host1x_debug_cont(struct output *o, const char *fmt, ...) int len; va_start(args, fmt); - len = vsnprintf(o->buf, sizeof(o->buf), fmt, args); + len = vscnprintf(o->buf, sizeof(o->buf), fmt, args); va_end(args); o->fn(o->ctx, o->buf, len, true); diff --git a/drivers/gpu/host1x/hw/channel_hw.c b/drivers/gpu/host1x/hw/channel_hw.c index 2df6a16d484e04..9dda73199889c2 100644 --- a/drivers/gpu/host1x/hw/channel_hw.c +++ b/drivers/gpu/host1x/hw/channel_hw.c @@ -36,10 +36,9 @@ static void trace_write_gather(struct host1x_cdma *cdma, struct host1x_bo *bo, for (i = 0; i < words; i += TRACE_MAX_LENGTH) { u32 num_words = min(words - i, TRACE_MAX_LENGTH); - offset += i * sizeof(u32); - trace_host1x_cdma_push_gather(dev_name(dev), bo, - num_words, offset, + num_words, + offset + i * sizeof(u32), mem); } diff --git a/drivers/gpu/nova-core/fb/hal/gb100.rs b/drivers/gpu/nova-core/fb/hal/gb100.rs index 6e0eba101ca154..51bd0c0bede696 100644 --- a/drivers/gpu/nova-core/fb/hal/gb100.rs +++ b/drivers/gpu/nova-core/fb/hal/gb100.rs @@ -78,6 +78,7 @@ fn write_sysmem_flush_page_gb100(bar: Bar0<'_>, addr: Bounded) { ); } +// This PMU reservation size is r570-specific. pub(super) const fn pmu_reserved_size_gb100() -> u32 { usize_into_u32::<{ const_align_up(SZ_8M + SZ_16M + SZ_4K, Alignment::new::()).unwrap() }>( ) diff --git a/drivers/gpu/nova-core/fb/hal/gb202.rs b/drivers/gpu/nova-core/fb/hal/gb202.rs index 038d1278c6341e..9978a5d2dec1c2 100644 --- a/drivers/gpu/nova-core/fb/hal/gb202.rs +++ b/drivers/gpu/nova-core/fb/hal/gb202.rs @@ -83,6 +83,7 @@ impl FbHal for Gb202 { fn non_wpr_heap_size(&self) -> u32 { // Non-WPR heap for GB20x (see Open RM: kgspGetNonWprHeapSize, GB202+). + // This size is r570-specific. u32::SZ_2M + u32::SZ_128K } diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index 8fc243c66e356b..04af98d2c1f810 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -57,12 +57,35 @@ struct NvdmPayloadCommandResponse { error_code: u32, } -/// Complete FSP response structure with MCTP and NVDM headers. +/// Common MCTP and NVDM headers shared by all FSP messages. #[repr(C, packed)] #[derive(Clone, Copy)] -struct FspResponse { +struct FspMessageHeader { mctp_header: MctpHeader, nvdm_header: NvdmHeader, +} + +// SAFETY: FspMessageHeader is a packed C struct with only integral fields. +unsafe impl AsBytes for FspMessageHeader {} + +// SAFETY: FspMessageHeader is a packed C struct with only integral fields. +unsafe impl FromBytes for FspMessageHeader {} + +impl FspMessageHeader { + /// Construct a standard FSP message header for the given NVDM type. + fn new(nvdm_type: NvdmType) -> Self { + Self { + mctp_header: MctpHeader::single_packet(), + nvdm_header: NvdmHeader::new(nvdm_type), + } + } +} + +/// Complete FSP response structure with MCTP and NVDM headers. +#[repr(C, packed)] +#[derive(Clone, Copy)] +struct FspResponse { + header: FspMessageHeader, response: NvdmPayloadCommandResponse, } @@ -94,29 +117,41 @@ struct NvdmPayloadCot { gsp_boot_args_sysmem_offset: u64, } -/// Complete FSP message structure with MCTP and NVDM headers. +/// Complete FSP COT (Chain of Trust) message structure. #[repr(C)] #[derive(Clone, Copy)] -struct FspMessage { - mctp_header: MctpHeader, - nvdm_header: NvdmHeader, +struct FspCotMessage { + header: FspMessageHeader, cot: NvdmPayloadCot, } -impl FspMessage { - /// Returns an in-place initializer for [`FspMessage`]. +impl FspCotMessage { + /// Computes the FRTS vidmem offset for the Chain-of-Trust message. It is measured backwards + /// from the end of the framebuffer. + fn frts_vidmem_offset(hal: &dyn hal::FspHal, fb_layout: &FbLayout) -> Result { + let mut offset = hal.fb_end_reserved_size(); + + // As per OpenRM's `kfspPrepareBootCommands_GH100`. + if fb_layout.pmu_reserved_size != 0 { + offset = (offset + u64::from(fb_layout.pmu_reserved_size)) + // The 2 MiB alignment is r570-specific. + .align_up(Alignment::new::()) + .ok_or(EINVAL)?; + } + + Ok(offset) + } + + /// Returns an in-place initializer for [`FspCotMessage`]. fn new<'a>( fb_layout: &FbLayout, fsp_fw: &'a FspFirmware, args: &'a FmcBootArgs, ) -> Result + 'a> { - // frts_offset is relative to FB end: FRTS_location = FB_END - frts_offset - let frts_vidmem_offset = if !args.resume { - let frts_reserved_size = fb_layout.heap.len() + u64::from(fb_layout.pmu_reserved_size); + let hal = hal::fsp_hal(args.chipset).ok_or(ENOTSUPP)?; - frts_reserved_size - .align_up(Alignment::new::()) - .ok_or(EINVAL)? + let frts_vidmem_offset = if !args.resume { + Self::frts_vidmem_offset(hal, fb_layout)? } else { 0 }; @@ -127,12 +162,11 @@ impl FspMessage { 0 }; - let version = hal::fsp_hal(args.chipset).ok_or(ENOTSUPP)?.cot_version(); + let version = hal.cot_version(); let size = num::usize_into_u16::<{ core::mem::size_of::() }>(); Ok(init!(Self { - mctp_header: MctpHeader::single_packet(), - nvdm_header: NvdmHeader::new(NvdmType::Cot), + header: FspMessageHeader::new(NvdmType::Cot), // The payload is packed, so we cannot use `init!`. Initialize it member-by-member using // `chain`. cot <- pin_init::init_zeroed(), @@ -143,8 +177,8 @@ impl FspMessage { msg.cot.gsp_fmc_sysmem_offset = fsp_fw.fmc_image.dma_handle(); msg.cot.frts_vidmem_offset = frts_vidmem_offset; msg.cot.frts_vidmem_size = frts_size; - // frts_sysmem_* intentionally left at zero for now, but will be needed for e.g. - // systems without VRAM. + // frts_sysmem_* are left at zero because this path places FRTS in vidmem. The sysmem + // fields point to an FRTS buffer in sysmem instead, for systems without VRAM. msg.cot.gsp_boot_args_sysmem_offset = args.fmc_boot_params.dma_handle(); msg.cot.sigs = *fsp_fw.fmc_sigs; @@ -153,11 +187,11 @@ impl FspMessage { } } -// SAFETY: `FspMessage` is `#[repr(C)]` with no padding, so all of its +// SAFETY: `FspCotMessage` is `#[repr(C)]` with no padding, so all of its // bytes are initialized. -unsafe impl AsBytes for FspMessage {} +unsafe impl AsBytes for FspCotMessage {} -impl MessageToFsp for FspMessage { +impl MessageToFsp for FspCotMessage { const NVDM_TYPE: NvdmType = NvdmType::Cot; } @@ -251,8 +285,8 @@ impl Fsp { EIO })?; - let mctp_header = response.mctp_header; - let nvdm_header = response.nvdm_header; + let mctp_header = response.header.mctp_header; + let nvdm_header = response.header.nvdm_header; let command_nvdm_type = response.response.command_nvdm_type; let error_code = response.response.error_code; @@ -310,7 +344,10 @@ impl Fsp { ) -> Result { dev_dbg!(dev, "Starting FSP boot sequence for {}\n", args.chipset); - let msg = KBox::init(FspMessage::new(fb_layout, &self.fsp_fw, args)?, GFP_KERNEL)?; + let msg = KBox::init( + FspCotMessage::new(fb_layout, &self.fsp_fw, args)?, + GFP_KERNEL, + )?; self.send_sync_fsp(dev, bar, &*msg)?; diff --git a/drivers/gpu/nova-core/fsp/hal.rs b/drivers/gpu/nova-core/fsp/hal.rs index b6f2624bb13d44..eaf5837ac5a886 100644 --- a/drivers/gpu/nova-core/fsp/hal.rs +++ b/drivers/gpu/nova-core/fsp/hal.rs @@ -19,6 +19,10 @@ pub(super) trait FspHal { /// Returns the FSP Chain of Trust protocol version this chipset advertises. fn cot_version(&self) -> u16; + + // TODO: consider moving this into the TLV firmware metadata when ready + /// Returns the size reserved at the end of the framebuffer, in bytes. + fn fb_end_reserved_size(&self) -> u64; } /// Returns the FSP HAL, or `None` if the architecture doesn't support FSP. diff --git a/drivers/gpu/nova-core/fsp/hal/gb100.rs b/drivers/gpu/nova-core/fsp/hal/gb100.rs index 42f5ecfc64004a..7cf53aa3d1ff09 100644 --- a/drivers/gpu/nova-core/fsp/hal/gb100.rs +++ b/drivers/gpu/nova-core/fsp/hal/gb100.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +use kernel::sizes::SizeConstants; + use crate::{ driver::Bar0, fsp::hal::FspHal, // @@ -17,6 +19,10 @@ impl FspHal for Gb100 { fn cot_version(&self) -> u16 { 2 } + + fn fb_end_reserved_size(&self) -> u64 { + u64::SZ_2M + u64::SZ_128K + } } const GB100: Gb100 = Gb100; diff --git a/drivers/gpu/nova-core/fsp/hal/gb202.rs b/drivers/gpu/nova-core/fsp/hal/gb202.rs index 1091b169a645a5..e380bbc5d58d7b 100644 --- a/drivers/gpu/nova-core/fsp/hal/gb202.rs +++ b/drivers/gpu/nova-core/fsp/hal/gb202.rs @@ -1,7 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -use kernel::io::Io; +use kernel::{ + io::Io, + sizes::SizeConstants, // +}; use crate::{ driver::Bar0, @@ -21,6 +24,10 @@ impl FspHal for Gb202 { fn cot_version(&self) -> u16 { 2 } + + fn fb_end_reserved_size(&self) -> u64 { + u64::SZ_2M + u64::SZ_128K + } } const GB202: Gb202 = Gb202; diff --git a/drivers/gpu/nova-core/fsp/hal/gh100.rs b/drivers/gpu/nova-core/fsp/hal/gh100.rs index 291acaf2845a7e..9a8563799da85d 100644 --- a/drivers/gpu/nova-core/fsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/fsp/hal/gh100.rs @@ -1,7 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -use kernel::io::Io; +use kernel::{ + io::Io, + sizes::SizeConstants, // +}; use crate::{ driver::Bar0, @@ -26,6 +29,10 @@ impl FspHal for Gh100 { fn cot_version(&self) -> u16 { 1 } + + fn fb_end_reserved_size(&self) -> u64 { + u64::SZ_2M + } } const GH100: Gh100 = Gh100; diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index b3c91731db45d4..b603b0bd2692b3 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -285,10 +285,10 @@ pub(crate) struct Gpu<'gpu> { } impl<'gpu> Gpu<'gpu> { - pub(crate) fn new( - pdev: &'gpu pci::Device>, + pub(crate) fn new<'a>( + pdev: &'gpu pci::Device>, bar: Bar0<'gpu>, - ) -> impl PinInit + 'gpu { + ) -> impl PinInit + use<'gpu, 'a> { try_pin_init!(Self { device: pdev.as_ref(), spec: Spec::new(pdev.as_ref(), bar).inspect(|spec| { diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 8afb62d689cb48..fbf81eb6c34dca 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -146,7 +146,7 @@ impl super::Gsp { self.cmdq .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev, chipset))?; self.cmdq - .send_command_no_wait(bar, commands::SetRegistry::new())?; + .send_command_no_wait(bar, commands::SetRegistry::new()?)?; hal.post_boot(&self, dev, bar, &gsp_fw, gsp_falcon, sec2_falcon)?; diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index f84de9f4f04503..0c2919b0980d86 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -66,37 +66,44 @@ struct RegistryEntry { /// The `SetRegistry` command. pub(crate) struct SetRegistry { - entries: [RegistryEntry; Self::NUM_ENTRIES], + entries: KVec, } impl SetRegistry { - // For now we hard-code the registry entries. Future work will allow others to - // be added as module parameters. - const NUM_ENTRIES: usize = 3; - /// Creates a new `SetRegistry` command, using a set of hardcoded entries. - pub(crate) fn new() -> Self { - Self { - entries: [ - // RMSecBusResetEnable - enables PCI secondary bus reset - RegistryEntry { - key: "RMSecBusResetEnable", - value: 1, - }, - // RMForcePcieConfigSave - forces GSP-RM to preserve PCI configuration registers on - // any PCI reset. - RegistryEntry { - key: "RMForcePcieConfigSave", - value: 1, - }, - // RMDevidCheckIgnore - allows GSP-RM to boot even if the PCI dev ID is not found - // in the internal product name database. - RegistryEntry { - key: "RMDevidCheckIgnore", - value: 1, - }, - ], - } + pub(crate) fn new() -> Result { + let mut entries = KVec::new(); + + // RMSecBusResetEnable - enables PCI secondary bus reset + entries.push( + RegistryEntry { + key: "RMSecBusResetEnable", + value: 1, + }, + GFP_KERNEL, + )?; + + // RMForcePcieConfigSave - forces GSP-RM to preserve PCI configuration registers on + // any PCI reset. + entries.push( + RegistryEntry { + key: "RMForcePcieConfigSave", + value: 1, + }, + GFP_KERNEL, + )?; + + // RMDevidCheckIgnore - allows GSP-RM to boot even if the PCI dev ID is not found + // in the internal product name database. + entries.push( + RegistryEntry { + key: "RMDevidCheckIgnore", + value: 1, + }, + GFP_KERNEL, + )?; + + Ok(Self { entries }) } } @@ -107,15 +114,15 @@ impl CommandToGsp for SetRegistry { type InitError = Infallible; fn init(&self) -> impl Init { - Self::Command::init(Self::NUM_ENTRIES as u32, self.variable_payload_len() as u32) + Self::Command::init(self.entries.len() as u32, self.size() as u32) } fn variable_payload_len(&self) -> usize { let mut key_size = 0; - for i in 0..Self::NUM_ENTRIES { - key_size += self.entries[i].key.len() + 1; // +1 for NULL terminator + for entry in self.entries.iter() { + key_size += entry.key.len() + 1; // +1 for NULL terminator } - Self::NUM_ENTRIES * size_of::() + key_size + self.entries.len() * size_of::() + key_size } fn init_variable_payload( @@ -123,12 +130,12 @@ impl CommandToGsp for SetRegistry { dst: &mut SBufferIter>, ) -> Result { let string_data_start_offset = size_of::() - + Self::NUM_ENTRIES * size_of::(); + + self.entries.len() * size_of::(); // Array for string data. let mut string_data = KVec::new(); - for entry in self.entries.iter().take(Self::NUM_ENTRIES) { + for entry in self.entries.iter() { dst.write_all( fw::commands::PackedRegistryEntry::new( (string_data_start_offset + string_data.len()) as u32, diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index 2f6301af71131b..eb7166148cc9a8 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -134,11 +134,19 @@ impl UnloadBundle for Sec2UnloadBundle { sec2_falcon: &Falcon, ) -> Result { // Run FWSEC-SB to reset the GSP falcon to its pre-libos state. - self.fwsec_sb.run(dev, bar, gsp_falcon)?; + // Log errors but keep going if it fails. + let fwsec_sb_res = self + .fwsec_sb + .run(dev, bar, gsp_falcon) + .inspect_err(|e| dev_err!(dev, "FWSEC-SB failed to run: {:?}\n", e)); // Remove WPR2 region if set. let wpr2_hi = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI); - if wpr2_hi.is_wpr2_set() { + let booter_unloader_res = (|| { + if !wpr2_hi.is_wpr2_set() { + return Ok(()); + } + sec2_falcon.reset(bar)?; sec2_falcon.load(dev, bar, &self.booter_unloader)?; @@ -160,9 +168,12 @@ impl UnloadBundle for Sec2UnloadBundle { ); return Err(EBUSY); } - } - Ok(()) + Ok(()) + })() + .inspect_err(|e| dev_err!(dev, "Booter Unloader failed to run: {:?}\n", e)); + + fwsec_sb_res.and(booter_unloader_res) } } diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index 0f49c1ab83ad43..a1af4a8bd2365a 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -570,7 +570,7 @@ register! { /// GA102 and later. pub(crate) NV_PRISCV_RISCV_CPUCTL(u32) @ PFalcon2Base + 0x00000388 { 7:7 active_stat => bool; - 0:0 halted => bool; + 4:4 halted => bool; } /// GA102 and later. diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c index befa990b3210e7..c22c1021ecc388 100644 --- a/drivers/hid/hid-asus.c +++ b/drivers/hid/hid-asus.c @@ -109,11 +109,36 @@ MODULE_DESCRIPTION("Asus HID Keyboard and TouchPad"); #define TRKID_SGN ((TRKID_MAX + 1) >> 1) -struct asus_kbd_leds { - struct asus_hid_listener listener; +enum asus_work_action_type { + FN_LOCK_SYNC, + BRIGHTNESS_SET, + WMI_FAN, +}; + +struct hid_raw_event_data { + u8 report_data[FEATURE_KBD_REPORT_SIZE]; + size_t report_size; +}; + +struct asus_work_action { + struct list_head node; + enum asus_work_action_type type; + union { + /* Data for BRIGHTNESS_SET */ + unsigned int brightness; + + /* Data for FN_LOCK_SYNC */ + bool fn_lock; + + /* Data for WMI_FAN */ + struct hid_raw_event_data fan_hid_data; + } data; +}; + +struct asus_worker { struct hid_device *hdev; struct work_struct work; - unsigned int brightness; + struct list_head actions; spinlock_t lock; bool removed; }; @@ -133,7 +158,8 @@ struct asus_drvdata { struct hid_device *hdev; struct input_dev *input; struct input_dev *tp_kbd_input; - struct asus_kbd_leds *kbd_backlight; + struct asus_worker *worker; + unsigned int kbd_backlight_brightness; const struct asus_touchpad_info *tp; struct power_supply *battery; struct power_supply_desc battery_desc; @@ -141,7 +167,7 @@ struct asus_drvdata { int battery_stat; bool battery_in_query; unsigned long battery_next_query; - struct work_struct fn_lock_sync_work; + struct asus_hid_listener listener; bool fn_lock; }; @@ -211,6 +237,29 @@ static const u8 asus_report_id_init[] = { FEATURE_KBD_LED_REPORT_ID2 }; +/* + * Send events to asus-wmi driver for handling special keys + */ +static int asus_wmi_send_event(struct asus_drvdata *drvdata, u8 code) +{ + int err; + u32 retval; + + err = asus_wmi_evaluate_method(ASUS_WMI_METHODID_DEVS, + ASUS_WMI_METHODID_NOTIF, code, &retval); + if (err) { + pr_warn("Failed to notify asus-wmi: %d\n", err); + return err; + } + + if (retval != 0) { + pr_warn("Failed to notify asus-wmi (retval): 0x%x\n", retval); + return -EIO; + } + + return 0; +} + static void asus_report_contact_down(struct asus_drvdata *drvdat, int toolType, u8 *data) { @@ -331,25 +380,71 @@ static int asus_e1239t_event(struct asus_drvdata *drvdat, u8 *data, int size) } /* - * Send events to asus-wmi driver for handling special keys + * Used in atomic contexts to schedule work involving sleeps operations or + * asus-wmi interactions. + * + * Caller is responsible to store relevant data in the structure to carry out + * the required action. + * + * This function must be called while the spin lock protecting the workqueue + * is already being held. */ -static int asus_wmi_send_event(struct asus_drvdata *drvdata, u8 code) +static void asus_worker_schedule(struct asus_worker *worker, struct asus_work_action *action) { - int err; - u32 retval; - - err = asus_wmi_evaluate_method(ASUS_WMI_METHODID_DEVS, - ASUS_WMI_METHODID_NOTIF, code, &retval); - if (err) { - pr_warn("Failed to notify asus-wmi: %d\n", err); - return err; + if (worker->removed) { + kfree(action); + return; } - if (retval != 0) { - pr_warn("Failed to notify asus-wmi (retval): 0x%x\n", retval); - return -EIO; + list_add_tail(&action->node, &worker->actions); + schedule_work(&worker->work); +} + +static int asus_kbd_fn_lock_set(struct asus_drvdata *drvdata, bool enabled) +{ + struct asus_work_action *action; + unsigned long flags; + + action = kzalloc(sizeof(struct asus_work_action), GFP_ATOMIC); + if (!action) + return -ENOMEM; + + drvdata->fn_lock = enabled; + action->type = FN_LOCK_SYNC; + action->data.fn_lock = drvdata->fn_lock; + INIT_LIST_HEAD(&action->node); + + spin_lock_irqsave(&drvdata->worker->lock, flags); + asus_worker_schedule(drvdata->worker, action); + spin_unlock_irqrestore(&drvdata->worker->lock, flags); + + return 0; +} + +static int asus_kbd_wmi_fan_send(struct asus_drvdata *drvdata, u8 *report_data, + size_t report_size) +{ + struct asus_work_action *action; + unsigned long flags; + + if (report_size > FEATURE_KBD_REPORT_SIZE) { + hid_err(drvdata->hdev, "Invalid report size for fan event: %zu\n", report_size); + return -EINVAL; } + action = kzalloc(sizeof(struct asus_work_action), GFP_NOWAIT); + if (!action) + return -ENOMEM; + + action->type = WMI_FAN; + action->data.fan_hid_data.report_size = report_size; + memcpy(action->data.fan_hid_data.report_data, report_data, report_size); + INIT_LIST_HEAD(&action->node); + + spin_lock_irqsave(&drvdata->worker->lock, flags); + asus_worker_schedule(drvdata->worker, action); + spin_unlock_irqrestore(&drvdata->worker->lock, flags); + return 0; } @@ -357,6 +452,7 @@ static int asus_event(struct hid_device *hdev, struct hid_field *field, struct hid_usage *usage, __s32 value) { struct asus_drvdata *drvdata = hid_get_drvdata(hdev); + int ret; if ((usage->hid & HID_USAGE_PAGE) == HID_UP_ASUSVENDOR && (usage->hid & HID_USAGE) != 0x00 && @@ -375,8 +471,11 @@ static int asus_event(struct hid_device *hdev, struct hid_field *field, return !asus_hid_event(ASUS_EV_BRTTOGGLE); case KEY_FN_ESC: if (drvdata->quirks & QUIRK_HID_FN_LOCK) { - drvdata->fn_lock = !drvdata->fn_lock; - schedule_work(&drvdata->fn_lock_sync_work); + ret = asus_kbd_fn_lock_set(drvdata, !drvdata->fn_lock); + if (ret) { + hid_err(hdev, "Error while toggling FN lock: %d\n", ret); + return ret; + } } break; } @@ -389,6 +488,12 @@ static int asus_raw_event(struct hid_device *hdev, struct hid_report *report, u8 *data, int size) { struct asus_drvdata *drvdata = hid_get_drvdata(hdev); + int ret; + + if (size < 2) { + hid_dbg(hdev, "Unexpected keyboard report size %d\n", size); + return 0; + } if (drvdata->battery && data[0] == BATTERY_REPORT_ID) return asus_report_battery(drvdata, data, size); @@ -414,19 +519,13 @@ static int asus_raw_event(struct hid_device *hdev, * pass to userspace so it can implement its own fan control. */ if (data[1] == ASUS_FAN_CTRL_KEY_CODE) { - int ret = asus_wmi_send_event(drvdata, ASUS_FAN_CTRL_KEY_CODE); + ret = asus_kbd_wmi_fan_send(drvdata, data, size); - if (ret == 0) { - /* Successfully handled by asus-wmi, block event */ + /* if execution deferred successfully block event */ + if (ret == 0) return -1; - } - /* - * Warn if asus-wmi failed (but not if it's unavailable). - * Let the event reach userspace in all failure cases. - */ - if (ret != -ENODEV) - hid_warn(hdev, "Failed to notify asus-wmi: %d\n", ret); + return ret; } /* @@ -569,59 +668,157 @@ static int asus_kbd_disable_oobe(struct hid_device *hdev) return 0; } -static int asus_kbd_set_fn_lock(struct hid_device *hdev, bool enabled) +static void asus_kbd_set_fn_lock(struct hid_device *hdev, bool enabled) { - u8 buf[] = { FEATURE_KBD_REPORT_ID, 0xd0, 0x4e, !!enabled }; + const u8 buf[FEATURE_KBD_REPORT_SIZE] = { FEATURE_KBD_REPORT_ID, 0xd0, 0x4e, !!enabled }; + int ret; - return asus_kbd_set_report(hdev, buf, sizeof(buf)); + ret = asus_kbd_set_report(hdev, buf, sizeof(buf)); + if (ret < 0) + hid_err(hdev, "Asus failed to set fn lock: %d\n", ret); } -static void asus_sync_fn_lock(struct work_struct *work) +static void asus_kbd_set_brightness(struct hid_device *hdev, u8 brightness) { - struct asus_drvdata *drvdata = - container_of(work, struct asus_drvdata, fn_lock_sync_work); + const u8 buf[FEATURE_KBD_REPORT_SIZE] = { + FEATURE_KBD_REPORT_ID, 0xba, 0xc5, 0xc4, brightness + }; + int ret; - asus_kbd_set_fn_lock(drvdata->hdev, drvdata->fn_lock); + ret = asus_kbd_set_report(hdev, buf, sizeof(buf)); + if (ret < 0) + hid_err(hdev, "Asus failed to set keyboard backlight: %d\n", ret); } -static void asus_schedule_work(struct asus_kbd_leds *led) +static void asus_kbd_wmi_fan(struct hid_device *hdev, struct hid_raw_event_data *data) { + struct asus_drvdata *drvdata = hid_get_drvdata(hdev); + int ret; + + ret = asus_wmi_send_event(drvdata, ASUS_FAN_CTRL_KEY_CODE); + + /* + * Warn if asus-wmi failed (but not if it's unavailable). + * Let the event reach userspace in all failure cases. + */ + switch (ret) { + case -ENODEV: + break; + case 0: + return; + default: + hid_warn(hdev, "Failed to notify asus-wmi: %d\n", ret); + break; + } + + /* + * Fallback: pass the raw event to the HID core; to avoid + * racing against the hid_report_raw_event() that generated + * this event use the same locking mechanism and wait for + * that function to terminate and signal the deferred execution + * before raising the stored event. + */ + down(&hdev->driver_input_lock); + hid_report_raw_event(hdev, HID_INPUT_REPORT, + data->report_data, data->report_size, + data->report_size, 1); + up(&hdev->driver_input_lock); +} + +static void asus_kbd_backlight_set(struct asus_hid_listener *listener, int brightness) +{ + struct asus_drvdata *drvdata = container_of(listener, struct asus_drvdata, listener); + struct asus_worker *worker = drvdata->worker; + struct asus_work_action *action; unsigned long flags; - spin_lock_irqsave(&led->lock, flags); - if (!led->removed) - schedule_work(&led->work); - spin_unlock_irqrestore(&led->lock, flags); + drvdata->kbd_backlight_brightness = brightness; + + action = kzalloc(sizeof(struct asus_work_action), GFP_NOWAIT); + if (!action) + return; + + action->type = BRIGHTNESS_SET; + action->data.brightness = brightness; + INIT_LIST_HEAD(&action->node); + + spin_lock_irqsave(&worker->lock, flags); + asus_worker_schedule(worker, action); + spin_unlock_irqrestore(&worker->lock, flags); } -static void asus_kbd_backlight_set(struct asus_hid_listener *listener, - int brightness) +static void asus_work(struct work_struct *work) { - struct asus_kbd_leds *led = container_of(listener, struct asus_kbd_leds, - listener); + struct asus_worker *worker = container_of(work, struct asus_worker, work); + struct asus_work_action *action = NULL; unsigned long flags; - spin_lock_irqsave(&led->lock, flags); - led->brightness = brightness; - spin_unlock_irqrestore(&led->lock, flags); + /* Save the action to be performed and clear the flag */ + spin_lock_irqsave(&worker->lock, flags); + if (!list_empty(&worker->actions)) { + action = list_first_entry(&worker->actions, + struct asus_work_action, node); + list_del(&action->node); + } + spin_unlock_irqrestore(&worker->lock, flags); + + if (!action) + return; + + switch (action->type) { + case BRIGHTNESS_SET: + asus_kbd_set_brightness(worker->hdev, action->data.brightness); + break; + case FN_LOCK_SYNC: + asus_kbd_set_fn_lock(worker->hdev, action->data.fn_lock); + break; + case WMI_FAN: + asus_kbd_wmi_fan(worker->hdev, &action->data.fan_hid_data); + break; + default: + hid_err(worker->hdev, "Invalid action type: %d\n", action->type); + break; + } + + kfree(action); - asus_schedule_work(led); + /* Re-schedule if there are more pending actions */ + spin_lock_irqsave(&worker->lock, flags); + if (!list_empty(&worker->actions)) + schedule_work(&worker->work); + spin_unlock_irqrestore(&worker->lock, flags); } -static void asus_kbd_backlight_work(struct work_struct *work) +static int asus_worker_create(struct hid_device *hdev, struct asus_drvdata *drvdata) { - struct asus_kbd_leds *led = container_of(work, struct asus_kbd_leds, work); - u8 buf[] = { FEATURE_KBD_REPORT_ID, 0xba, 0xc5, 0xc4, 0x00 }; - int ret; + drvdata->worker = devm_kzalloc(&hdev->dev, sizeof(struct asus_worker), GFP_KERNEL); + if (!drvdata->worker) + return -ENOMEM; + + drvdata->worker->removed = false; + drvdata->worker->hdev = hdev; + INIT_LIST_HEAD(&drvdata->worker->actions); + + INIT_WORK(&drvdata->worker->work, asus_work); + spin_lock_init(&drvdata->worker->lock); + + return 0; +} + +static void asus_worker_stop(struct asus_worker *worker) +{ + struct asus_work_action *action, *tmp; unsigned long flags; - spin_lock_irqsave(&led->lock, flags); - buf[4] = led->brightness; - spin_unlock_irqrestore(&led->lock, flags); + spin_lock_irqsave(&worker->lock, flags); + worker->removed = true; + list_for_each_entry_safe(action, tmp, &worker->actions, node) { + list_del(&action->node); + kfree(action); + } + spin_unlock_irqrestore(&worker->lock, flags); - ret = asus_kbd_set_report(led->hdev, buf, sizeof(buf)); - if (ret < 0) - hid_err(led->hdev, "Asus failed to set keyboard backlight: %d\n", ret); + cancel_work_sync(&worker->work); } /* @@ -644,7 +841,7 @@ static int mcu_parse_version_string(const u8 *response, size_t response_size) dots++; } - if (dots != 2 || p >= end || (p + 3) >= end) + if (dots != 2 || end - p < 3) return -EINVAL; memcpy(buf, p, 3); @@ -760,23 +957,11 @@ static int asus_kbd_register_leds(struct hid_device *hdev) le16_to_cpu(udev->descriptor.idProduct)); } - drvdata->kbd_backlight = devm_kzalloc(&hdev->dev, - sizeof(struct asus_kbd_leds), - GFP_KERNEL); - if (!drvdata->kbd_backlight) - return -ENOMEM; - - drvdata->kbd_backlight->removed = false; - drvdata->kbd_backlight->brightness = 0; - drvdata->kbd_backlight->hdev = hdev; - drvdata->kbd_backlight->listener.brightness_set = asus_kbd_backlight_set; - INIT_WORK(&drvdata->kbd_backlight->work, asus_kbd_backlight_work); - spin_lock_init(&drvdata->kbd_backlight->lock); - - ret = asus_hid_register_listener(&drvdata->kbd_backlight->listener); + drvdata->listener.brightness_set = asus_kbd_backlight_set; + ret = asus_hid_register_listener(&drvdata->listener); if (ret < 0) { - /* No need to have this still around */ - devm_kfree(&hdev->dev, drvdata->kbd_backlight); + hid_err(hdev, "Unable to register kbd brightness listener: %d\n", ret); + drvdata->listener.brightness_set = NULL; } return ret; @@ -998,11 +1183,9 @@ static int asus_input_configured(struct hid_device *hdev, struct hid_input *hi) drvdata->input = input; - if (drvdata->quirks & QUIRK_HID_FN_LOCK) { - drvdata->fn_lock = true; - INIT_WORK(&drvdata->fn_lock_sync_work, asus_sync_fn_lock); - asus_kbd_set_fn_lock(hdev, true); - } + if ((drvdata->quirks & QUIRK_HID_FN_LOCK) && + (asus_kbd_fn_lock_set(drvdata, true))) + hid_warn(hdev, "Error while setting FN lock to ON\n"); return 0; } @@ -1165,20 +1348,16 @@ static int asus_start_multitouch(struct hid_device *hdev) static int __maybe_unused asus_resume(struct hid_device *hdev) { struct asus_drvdata *drvdata = hid_get_drvdata(hdev); - int ret = 0; - if (drvdata->kbd_backlight) { - const u8 buf[] = { FEATURE_KBD_REPORT_ID, 0xba, 0xc5, 0xc4, - drvdata->kbd_backlight->brightness }; - ret = asus_kbd_set_report(hdev, buf, sizeof(buf)); - if (ret < 0) { - hid_err(hdev, "Asus failed to set keyboard backlight: %d\n", ret); - goto asus_resume_err; - } - } + /* + * If we have a backlight listener registered, restore the previous state, + * in case of error do not fail: most models restore the backlight + * automatically, and the error is non-fatal. + */ + if (drvdata->listener.brightness_set) + asus_kbd_backlight_set(&drvdata->listener, drvdata->kbd_backlight_brightness); -asus_resume_err: - return ret; + return 0; } static int __maybe_unused asus_reset_resume(struct hid_device *hdev) @@ -1288,8 +1467,15 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id) is_vendor = true; } + ret = asus_worker_create(hdev, drvdata); + if (ret) { + hid_warn(hdev, "Failed to initialize worker: %d\n", ret); + return ret; + } + ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT); if (ret) { + asus_worker_stop(drvdata->worker); hid_err(hdev, "Asus hw start failed: %d\n", ret); return ret; } @@ -1337,6 +1523,10 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id) return 0; err_stop_hw: + if (drvdata->listener.brightness_set) + asus_hid_unregister_listener(&drvdata->listener); + + asus_worker_stop(drvdata->worker); hid_hw_stop(hdev); return ret; } @@ -1344,21 +1534,11 @@ err_stop_hw: static void asus_remove(struct hid_device *hdev) { struct asus_drvdata *drvdata = hid_get_drvdata(hdev); - unsigned long flags; - - if (drvdata->kbd_backlight) { - asus_hid_unregister_listener(&drvdata->kbd_backlight->listener); - - spin_lock_irqsave(&drvdata->kbd_backlight->lock, flags); - drvdata->kbd_backlight->removed = true; - spin_unlock_irqrestore(&drvdata->kbd_backlight->lock, flags); - - cancel_work_sync(&drvdata->kbd_backlight->work); - } - if (drvdata->quirks & QUIRK_HID_FN_LOCK) - cancel_work_sync(&drvdata->fn_lock_sync_work); + if (drvdata->listener.brightness_set) + asus_hid_unregister_listener(&drvdata->listener); + asus_worker_stop(drvdata->worker); hid_hw_stop(hdev); } diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index d6fbc2111facdd..14a06ab435a237 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -2454,9 +2454,16 @@ EXPORT_SYMBOL_GPL(hid_hw_start); * * This is usually called from remove function or from probe when something * failed and hid_hw_start was called already. + * + * If the caller enabled HID input via hid_device_io_start() and is unwinding + * without an explicit hid_device_io_stop(), quiesce input first so that + * in-flight reports cannot reach handlers (e.g. hidraw_report_event) whose + * backing objects hid_disconnect() is about to free. */ void hid_hw_stop(struct hid_device *hdev) { + if (hdev->io_started) + hid_device_io_stop(hdev); hid_disconnect(hdev); hdev->ll_driver->stop(hdev); } @@ -2849,6 +2856,8 @@ static int __hid_device_probe(struct hid_device *hdev, struct hid_driver *hdrv) */ if (ret) { + if (hdev->io_started) + hid_device_io_stop(hdev); devres_release_group(&hdev->dev, hdev->devres_group_id); hid_close_report(hdev); hdev->driver = NULL; diff --git a/drivers/hid/hid-haptic.c b/drivers/hid/hid-haptic.c index deadab28cdbefa..66d90f0f78a3b7 100644 --- a/drivers/hid/hid-haptic.c +++ b/drivers/hid/hid-haptic.c @@ -187,7 +187,7 @@ static void fill_effect_buf(struct hid_haptic_device *haptic, value = waveform_ordinal; break; default: - break; + continue; } field->value[j] = value; diff --git a/drivers/hid/hid-lg4ff.c b/drivers/hid/hid-lg4ff.c index e901fdb7d033c6..62f99a93210c9d 100644 --- a/drivers/hid/hid-lg4ff.c +++ b/drivers/hid/hid-lg4ff.c @@ -336,6 +336,8 @@ int lg4ff_raw_event(struct hid_device *hdev, struct hid_report *report, if (entry->wdata.combine) { switch (entry->wdata.product_id) { case USB_DEVICE_ID_LOGITECH_WHEEL: + if (size < 7) + return 0; rd[5] = rd[3]; rd[6] = 0x7F; return 1; @@ -343,10 +345,14 @@ int lg4ff_raw_event(struct hid_device *hdev, struct hid_report *report, case USB_DEVICE_ID_LOGITECH_WINGMAN_FFG: case USB_DEVICE_ID_LOGITECH_MOMO_WHEEL: case USB_DEVICE_ID_LOGITECH_MOMO_WHEEL2: + if (size < 6) + return 0; rd[4] = rd[3]; rd[5] = 0x7F; return 1; case USB_DEVICE_ID_LOGITECH_DFP_WHEEL: + if (size < 7) + return 0; rd[5] = rd[4]; rd[6] = 0x7F; return 1; @@ -366,6 +372,8 @@ int lg4ff_raw_event(struct hid_device *hdev, struct hid_report *report, } /* Compute a combined axis when wheel does not supply it */ + if (size <= offset + 1) + return 0; rd[offset] = (0xFF + rd[offset] - rd[offset+1]) >> 1; rd[offset+1] = 0x7F; return 1; diff --git a/drivers/hid/hid-logitech-hidpp.c b/drivers/hid/hid-logitech-hidpp.c index 90b0184df777f6..fb2062233df228 100644 --- a/drivers/hid/hid-logitech-hidpp.c +++ b/drivers/hid/hid-logitech-hidpp.c @@ -2861,18 +2861,19 @@ static int hidpp_ff_init(struct hidpp_device *hidpp, * ownership to FF core */ data = kmemdup(data, sizeof(*data), GFP_KERNEL); - if (!data) - return -ENOMEM; + if (!data) { + error = -ENOMEM; + goto err_destroy_ff; + } data->effect_ids = kzalloc_objs(int, num_slots); if (!data->effect_ids) { - kfree(data); - return -ENOMEM; + error = -ENOMEM; + goto err_free_data; } data->wq = create_singlethread_workqueue("hidpp-ff-sendqueue"); if (!data->wq) { - kfree(data->effect_ids); - kfree(data); - return -ENOMEM; + error = -ENOMEM; + goto err_free_effect_ids; } data->hidpp = hidpp; @@ -2902,6 +2903,14 @@ static int hidpp_ff_init(struct hidpp_device *hidpp, version); return 0; + +err_free_effect_ids: + kfree(data->effect_ids); +err_free_data: + kfree(data); +err_destroy_ff: + input_ff_destroy(dev); + return error; } /* ************************************************************************** */ diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c index edb37b4c867e4b..571166a769b904 100644 --- a/drivers/hid/hid-multitouch.c +++ b/drivers/hid/hid-multitouch.c @@ -2722,7 +2722,7 @@ static const struct hid_device_id mt_devices[] = { HID_ANY_ID) }, /* Hantick */ - { .driver_data = MT_CLS_NSMU, + { .driver_data = MT_CLS_WIN_8_FORCE_MULTI_INPUT_NSMU, HID_DEVICE(BUS_I2C, HID_GROUP_MULTITOUCH_WIN_8, I2C_VENDOR_ID_HANTICK, I2C_PRODUCT_ID_HANTICK_5288) }, diff --git a/drivers/hid/hid-nintendo.c b/drivers/hid/hid-nintendo.c index f3c8a4a3640027..4ee402e2f9cab6 100644 --- a/drivers/hid/hid-nintendo.c +++ b/drivers/hid/hid-nintendo.c @@ -1474,7 +1474,6 @@ static void joycon_parse_imu_report(struct joycon_ctlr *ctlr, dropped_threshold = ctlr->imu_avg_delta_ms * 3 / 2; dropped_pkts = (delta - min(delta, dropped_threshold)) / ctlr->imu_avg_delta_ms; - ctlr->imu_timestamp_us += 1000 * ctlr->imu_avg_delta_ms; if (dropped_pkts > JC_IMU_DROPPED_PKT_WARNING) { hid_warn_ratelimited(ctlr->hdev, "compensating for %u dropped IMU reports\n", diff --git a/drivers/hid/hid-roccat-kone.c b/drivers/hid/hid-roccat-kone.c index 58654cf78f0df3..17495fcc8b7da2 100644 --- a/drivers/hid/hid-roccat-kone.c +++ b/drivers/hid/hid-roccat-kone.c @@ -36,6 +36,8 @@ static uint profile_numbers[5] = {0, 1, 2, 3, 4}; static void kone_profile_activated(struct kone_device *kone, uint new_profile) { + if (new_profile < 1 || new_profile > ARRAY_SIZE(kone->profiles)) + new_profile = 1; kone->actual_profile = new_profile; kone->actual_dpi = kone->profiles[new_profile - 1].startup_dpi; } @@ -793,8 +795,10 @@ static void kone_keep_values_up_to_date(struct kone_device *kone, { switch (event->event) { case kone_mouse_event_switch_profile: - kone->actual_dpi = kone->profiles[event->value - 1]. - startup_dpi; + if (event->value >= 1 && + event->value <= ARRAY_SIZE(kone->profiles)) + kone->actual_dpi = + kone->profiles[event->value - 1].startup_dpi; fallthrough; case kone_mouse_event_osd_profile: kone->actual_profile = event->value; diff --git a/drivers/hid/hid-sony.c b/drivers/hid/hid-sony.c index af1ad2c2337079..a8b3a1039f0caa 100644 --- a/drivers/hid/hid-sony.c +++ b/drivers/hid/hid-sony.c @@ -504,7 +504,7 @@ struct motion_output_report_02 { u8 r, g, b; u8 zero2; u8 rumble; -}; +} __packed; static_assert(sizeof(struct motion_output_report_02) == 7); #define SIXAXIS_REPORT_0xF2_SIZE 17 diff --git a/drivers/hid/hid-steam.c b/drivers/hid/hid-steam.c index 197126d6e08100..80d2eabc930eca 100644 --- a/drivers/hid/hid-steam.c +++ b/drivers/hid/hid-steam.c @@ -43,6 +43,7 @@ #include #include #include +#include #include "hid-ids.h" MODULE_DESCRIPTION("HID driver for Valve Steam Controller"); @@ -68,13 +69,14 @@ static LIST_HEAD(steam_devices); /* Joystick runs are about 5 mm and 32768 units */ #define STEAM_DECK_JOYSTICK_RESOLUTION 6553 /* Accelerometer has 16 bit resolution and a range of +/- 2g */ -#define STEAM_DECK_ACCEL_RES_PER_G 16384 -#define STEAM_DECK_ACCEL_RANGE 32768 +#define STEAM_ACCEL_RES_PER_G 16384 +#define STEAM_ACCEL_RANGE 32768 +#define STEAM_ACCEL_FUZZ 128 #define STEAM_DECK_ACCEL_FUZZ 32 /* Gyroscope has 16 bit resolution and a range of +/- 2000 dps */ -#define STEAM_DECK_GYRO_RES_PER_DPS 16 -#define STEAM_DECK_GYRO_RANGE 32768 -#define STEAM_DECK_GYRO_FUZZ 1 +#define STEAM_GYRO_RES_PER_DPS 16 +#define STEAM_GYRO_RANGE 32768 +#define STEAM_GYRO_FUZZ 0 #define STEAM_PAD_FUZZ 256 @@ -149,7 +151,7 @@ enum { SETTING_USB_DEBUG_MODE, SETTING_LEFT_TRACKPAD_MODE, SETTING_RIGHT_TRACKPAD_MODE, - SETTING_MOUSE_POINTER_ENABLED, + SETTING_LIZARD_MODE, /* 10 */ SETTING_DPAD_DEADZONE, @@ -243,14 +245,39 @@ enum { /* Input report identifiers */ enum { - ID_CONTROLLER_STATE = 1, - ID_CONTROLLER_DEBUG = 2, - ID_CONTROLLER_WIRELESS = 3, - ID_CONTROLLER_STATUS = 4, - ID_CONTROLLER_DEBUG2 = 5, - ID_CONTROLLER_SECONDARY_STATE = 6, - ID_CONTROLLER_BLE_STATE = 7, - ID_CONTROLLER_DECK_STATE = 9 + ID_CONTROLLER_STATE = 1, + ID_CONTROLLER_DEBUG = 2, + ID_CONTROLLER_WIRELESS = 3, + ID_CONTROLLER_STATUS = 4, + ID_CONTROLLER_DEBUG2 = 5, + ID_CONTROLLER_SECONDARY_STATE = 6, + ID_CONTROLLER_BLE_STATE = 7, + ID_CONTROLLER_DECK_STATE = 9, +}; + +/* Read-only attributes */ +enum { + ATTRIB_UNIQUE_ID, // deprecated + ATTRIB_PRODUCT_ID, + ATTRIB_PRODUCT_REVISON, // deprecated + ATTRIB_CAPABILITIES = ATTRIB_PRODUCT_REVISON, // intentional aliasing + ATTRIB_FIRMWARE_VERSION, // deprecated + ATTRIB_FIRMWARE_BUILD_TIME, + ATTRIB_RADIO_FIRMWARE_BUILD_TIME, + ATTRIB_RADIO_DEVICE_ID0, + ATTRIB_RADIO_DEVICE_ID1, + ATTRIB_DONGLE_FIRMWARE_BUILD_TIME, + ATTRIB_HW_ID, // AKA BOARD_REVISION, + ATTRIB_BOOTLOADER_BUILD_TIME, + ATTRIB_CONNECTION_INTERVAL_IN_US, + ATTRIB_SECONDARY_FIRMWARE_BUILD_TIME, + ATTRIB_SECONDARY_BOOTLOADER_BUILD_TIME, + ATTRIB_SECONDARY_HW_ID, // AKA BOARD_REVISION, + ATTRIB_STREAMING, + ATTRIB_TRACKPAD_ID, + ATTRIB_SECONDARY_TRACKPAD_ID, + + ATTRIB_COUNT }; /* String attribute identifiers */ @@ -259,14 +286,14 @@ enum { ATTRIB_STR_UNIT_SERIAL, }; -/* Values for GYRO_MODE (bitmask) */ +/* Values for IMU_MODE (bitmask) */ enum { - SETTING_GYRO_MODE_OFF = 0, - SETTING_GYRO_MODE_STEERING = BIT(0), - SETTING_GYRO_MODE_TILT = BIT(1), - SETTING_GYRO_MODE_SEND_ORIENTATION = BIT(2), - SETTING_GYRO_MODE_SEND_RAW_ACCEL = BIT(3), - SETTING_GYRO_MODE_SEND_RAW_GYRO = BIT(4), + SETTING_IMU_MODE_OFF = 0, + SETTING_IMU_MODE_STEERING = BIT(0), + SETTING_IMU_MODE_TILT = BIT(1), + SETTING_IMU_MODE_SEND_ORIENTATION = BIT(2), + SETTING_IMU_MODE_SEND_RAW_ACCEL = BIT(3), + SETTING_IMU_MODE_SEND_RAW_GYRO = BIT(4), }; /* Trackpad modes */ @@ -282,6 +309,11 @@ enum { TRACKPAD_GESTURE_KEYBOARD, }; +struct steam_controller_attribute { + unsigned char tag; + __le32 value; +} __packed; + /* Pad identifiers for the deck */ #define STEAM_PAD_LEFT 0 #define STEAM_PAD_RIGHT 1 @@ -313,6 +345,7 @@ struct steam_device { u16 rumble_left; u16 rumble_right; unsigned int sensor_timestamp_us; + unsigned int sensor_update_rate_us; struct work_struct unregister_work; }; @@ -323,6 +356,13 @@ static int steam_recv_report(struct steam_device *steam, u8 *buf; int ret; + /* + * All reports start with a two byte header. + * We must read at least two bytes to get a sensible output. + */ + if (size < 2) + return -EINVAL; + r = steam->hdev->report_enum[HID_FEATURE_REPORT].report_id_hash[0]; if (!r) { hid_err(steam->hdev, "No HID_FEATURE_REPORT submitted - nothing to read\n"); @@ -345,10 +385,31 @@ static int steam_recv_report(struct steam_device *steam, ret = hid_hw_raw_request(steam->hdev, 0x00, buf, hid_report_len(r) + 1, HID_FEATURE_REPORT, HID_REQ_GET_REPORT); - if (ret > 0) - memcpy(data, buf + 1, min(size, ret - 1)); + if (ret > 0) { + /* Remove the report ID from the return buffer */ + ret--; + size = min(size, ret); + memcpy(data, buf + 1, size); + } kfree(buf); - return ret; + + if (ret < 0) + hid_err(steam->hdev, "%s: error %d\n", __func__, ret); + else + hid_dbg(steam->hdev, "Received report %*ph\n", size, data); + if (ret < 0) + return ret; + + if (ret < 2) { + hid_err(steam->hdev, "%s: reply too short\n", __func__); + return -EPROTO; + } + if (ret < data[1] + 2) { + hid_err(steam->hdev, "%s: expected %u bytes, read %i\n", + __func__, data[1] + 2, ret); + return -EPROTO; + } + return size; } static int steam_send_report(struct steam_device *steam, @@ -375,6 +436,8 @@ static int steam_send_report(struct steam_device *steam, /* The report ID is always 0 */ memcpy(buf + 1, cmd, size); + hid_dbg(steam->hdev, "Sending report %*ph\n", size, cmd); + /* * Sometimes the wireless controller fails with EPIPE * when sending a feature report. @@ -447,25 +510,59 @@ static int steam_get_serial(struct steam_device *steam) u8 cmd[] = {ID_GET_STRING_ATTRIBUTE, sizeof(steam->serial_no), ATTRIB_STR_UNIT_SERIAL}; u8 reply[3 + STEAM_SERIAL_LEN + 1]; - mutex_lock(&steam->report_mutex); + guard(mutex)(&steam->report_mutex); ret = steam_send_report(steam, cmd, sizeof(cmd)); if (ret < 0) - goto out; + return ret; ret = steam_recv_report(steam, reply, sizeof(reply)); if (ret < 0) - goto out; + return ret; if (reply[0] != ID_GET_STRING_ATTRIBUTE || reply[1] < 1 || reply[1] > sizeof(steam->serial_no) || reply[2] != ATTRIB_STR_UNIT_SERIAL) { - ret = -EIO; - goto out; + hid_err(steam->hdev, "%s: invalid reply (%*ph)\n", __func__, + (int)sizeof(reply), reply); + return -EIO; } reply[3 + STEAM_SERIAL_LEN] = 0; strscpy(steam->serial_no, reply + 3, reply[1]); -out: - mutex_unlock(&steam->report_mutex); return ret; } +static int steam_get_attributes(struct steam_device *steam) +{ + int ret = 0; + u8 cmd[] = {ID_GET_ATTRIBUTES_VALUES, 0}; + u8 reply[64] = {}; + u8 size; + int i; + struct steam_controller_attribute *attr; + + guard(mutex)(&steam->report_mutex); + ret = steam_send_report(steam, cmd, sizeof(cmd)); + if (ret < 0) + return ret; + ret = steam_recv_report(steam, reply, sizeof(reply)); + if (ret < 0) + return ret; + if (reply[0] != ID_GET_ATTRIBUTES_VALUES || reply[1] < 2) { + hid_err(steam->hdev, "%s: invalid reply (%*ph)\n", __func__, + (int)sizeof(reply), reply); + return -EIO; + } + + size = min(reply[1], sizeof(reply) - 2); + for (i = 0; i + sizeof(*attr) <= size; i += sizeof(*attr)) { + attr = (struct steam_controller_attribute *)&reply[i + 2]; + if (attr->tag == ATTRIB_CONNECTION_INTERVAL_IN_US) { + steam->sensor_update_rate_us = get_unaligned_le32(&attr->value); + hid_dbg(steam->hdev, "Sensor update rate: %uus\n", + steam->sensor_update_rate_us); + } + } + + return 0; +} + /* * This command requests the wireless adaptor to post an event * with the connection status. Useful if this driver is loaded when @@ -473,11 +570,8 @@ out: */ static inline int steam_request_conn_status(struct steam_device *steam) { - int ret; - mutex_lock(&steam->report_mutex); - ret = steam_send_report_byte(steam, ID_DONGLE_GET_WIRELESS_STATE); - mutex_unlock(&steam->report_mutex); - return ret; + guard(mutex)(&steam->report_mutex); + return steam_send_report_byte(steam, ID_DONGLE_GET_WIRELESS_STATE); } /* @@ -624,6 +718,42 @@ static void steam_input_close(struct input_dev *dev) } } +static int steam_sensor_open(struct input_dev *dev) +{ + struct steam_device *steam = input_get_drvdata(dev); + unsigned long flags; + bool client_opened; + + spin_lock_irqsave(&steam->lock, flags); + client_opened = steam->client_opened; + spin_unlock_irqrestore(&steam->lock, flags); + if (client_opened) + return 0; + + guard(mutex)(&steam->report_mutex); + steam_write_settings(steam, SETTING_IMU_MODE, + SETTING_IMU_MODE_SEND_RAW_ACCEL | SETTING_IMU_MODE_SEND_RAW_GYRO, + 0); + + return 0; +} + +static void steam_sensor_close(struct input_dev *dev) +{ + struct steam_device *steam = input_get_drvdata(dev); + unsigned long flags; + bool client_opened; + + spin_lock_irqsave(&steam->lock, flags); + client_opened = steam->client_opened; + spin_unlock_irqrestore(&steam->lock, flags); + if (client_opened) + return; + + guard(mutex)(&steam->report_mutex); + steam_write_settings(steam, SETTING_IMU_MODE, 0, 0); +} + static enum power_supply_property steam_battery_props[] = { POWER_SUPPLY_PROP_PRESENT, POWER_SUPPLY_PROP_SCOPE, @@ -837,9 +967,6 @@ static int steam_sensors_register(struct steam_device *steam) struct input_dev *sensors; int ret; - if (!(steam->quirks & STEAM_QUIRK_DECK)) - return 0; - rcu_read_lock(); sensors = rcu_dereference(steam->sensors); rcu_read_unlock(); @@ -854,8 +981,14 @@ static int steam_sensors_register(struct steam_device *steam) input_set_drvdata(sensors, steam); sensors->dev.parent = &hdev->dev; + if (!(steam->quirks & STEAM_QUIRK_DECK)) { + sensors->open = steam_sensor_open; + sensors->close = steam_sensor_close; + } - sensors->name = "Steam Deck Motion Sensors"; + sensors->name = steam->quirks & STEAM_QUIRK_DECK ? + "Steam Deck Motion Sensors" : + "Steam Controller Motion Sensors"; sensors->phys = hdev->phys; sensors->uniq = steam->serial_no; sensors->id.bustype = hdev->bus; @@ -867,25 +1000,34 @@ static int steam_sensors_register(struct steam_device *steam) __set_bit(EV_MSC, sensors->evbit); __set_bit(MSC_TIMESTAMP, sensors->mscbit); - input_set_abs_params(sensors, ABS_X, -STEAM_DECK_ACCEL_RANGE, - STEAM_DECK_ACCEL_RANGE, STEAM_DECK_ACCEL_FUZZ, 0); - input_set_abs_params(sensors, ABS_Y, -STEAM_DECK_ACCEL_RANGE, - STEAM_DECK_ACCEL_RANGE, STEAM_DECK_ACCEL_FUZZ, 0); - input_set_abs_params(sensors, ABS_Z, -STEAM_DECK_ACCEL_RANGE, - STEAM_DECK_ACCEL_RANGE, STEAM_DECK_ACCEL_FUZZ, 0); - input_abs_set_res(sensors, ABS_X, STEAM_DECK_ACCEL_RES_PER_G); - input_abs_set_res(sensors, ABS_Y, STEAM_DECK_ACCEL_RES_PER_G); - input_abs_set_res(sensors, ABS_Z, STEAM_DECK_ACCEL_RES_PER_G); - - input_set_abs_params(sensors, ABS_RX, -STEAM_DECK_GYRO_RANGE, - STEAM_DECK_GYRO_RANGE, STEAM_DECK_GYRO_FUZZ, 0); - input_set_abs_params(sensors, ABS_RY, -STEAM_DECK_GYRO_RANGE, - STEAM_DECK_GYRO_RANGE, STEAM_DECK_GYRO_FUZZ, 0); - input_set_abs_params(sensors, ABS_RZ, -STEAM_DECK_GYRO_RANGE, - STEAM_DECK_GYRO_RANGE, STEAM_DECK_GYRO_FUZZ, 0); - input_abs_set_res(sensors, ABS_RX, STEAM_DECK_GYRO_RES_PER_DPS); - input_abs_set_res(sensors, ABS_RY, STEAM_DECK_GYRO_RES_PER_DPS); - input_abs_set_res(sensors, ABS_RZ, STEAM_DECK_GYRO_RES_PER_DPS); + if (steam->quirks & STEAM_QUIRK_DECK) { + input_set_abs_params(sensors, ABS_X, -STEAM_ACCEL_RANGE, + STEAM_ACCEL_RANGE, STEAM_DECK_ACCEL_FUZZ, 0); + input_set_abs_params(sensors, ABS_Y, -STEAM_ACCEL_RANGE, + STEAM_ACCEL_RANGE, STEAM_DECK_ACCEL_FUZZ, 0); + input_set_abs_params(sensors, ABS_Z, -STEAM_ACCEL_RANGE, + STEAM_ACCEL_RANGE, STEAM_DECK_ACCEL_FUZZ, 0); + } else { + input_set_abs_params(sensors, ABS_X, -STEAM_ACCEL_RANGE, + STEAM_ACCEL_RANGE, STEAM_ACCEL_FUZZ, 0); + input_set_abs_params(sensors, ABS_Y, -STEAM_ACCEL_RANGE, + STEAM_ACCEL_RANGE, STEAM_ACCEL_FUZZ, 0); + input_set_abs_params(sensors, ABS_Z, -STEAM_ACCEL_RANGE, + STEAM_ACCEL_RANGE, STEAM_ACCEL_FUZZ, 0); + } + input_abs_set_res(sensors, ABS_X, STEAM_ACCEL_RES_PER_G); + input_abs_set_res(sensors, ABS_Y, STEAM_ACCEL_RES_PER_G); + input_abs_set_res(sensors, ABS_Z, STEAM_ACCEL_RES_PER_G); + + input_set_abs_params(sensors, ABS_RX, -STEAM_GYRO_RANGE, + STEAM_GYRO_RANGE, STEAM_GYRO_FUZZ, 0); + input_set_abs_params(sensors, ABS_RY, -STEAM_GYRO_RANGE, + STEAM_GYRO_RANGE, STEAM_GYRO_FUZZ, 0); + input_set_abs_params(sensors, ABS_RZ, -STEAM_GYRO_RANGE, + STEAM_GYRO_RANGE, STEAM_GYRO_FUZZ, 0); + input_abs_set_res(sensors, ABS_RX, STEAM_GYRO_RES_PER_DPS); + input_abs_set_res(sensors, ABS_RY, STEAM_GYRO_RES_PER_DPS); + input_abs_set_res(sensors, ABS_RZ, STEAM_GYRO_RES_PER_DPS); ret = input_register_device(sensors); if (ret) @@ -916,9 +1058,6 @@ static void steam_sensors_unregister(struct steam_device *steam) { struct input_dev *sensors; - if (!(steam->quirks & STEAM_QUIRK_DECK)) - return; - rcu_read_lock(); sensors = rcu_dereference(steam->sensors); rcu_read_unlock(); @@ -966,6 +1105,12 @@ static int steam_register(struct steam_device *steam) strscpy(steam->serial_no, "XXXXXXXXXX", sizeof(steam->serial_no)); + ret = steam_get_attributes(steam); + if (ret < 0) + hid_err(steam->hdev, + "%s:steam_get_attributes failed with error %d\n", + __func__, ret); + hid_info(steam->hdev, "Steam Controller '%s' connected", steam->serial_no); @@ -1049,6 +1194,7 @@ static void steam_mode_switch_cb(struct work_struct *work) return; steam->gamepad_mode = !steam->gamepad_mode; + hid_dbg(steam->hdev, "%s: switching gamepad mode to %i\n", __func__, steam->gamepad_mode); if (steam->gamepad_mode) steam_set_lizard_mode(steam, false); else { @@ -1244,6 +1390,10 @@ static int steam_probe(struct hid_device *hdev, INIT_LIST_HEAD(&steam->list); INIT_WORK(&steam->rumble_work, steam_haptic_rumble_cb); steam->sensor_timestamp_us = 0; + if (steam->quirks & STEAM_QUIRK_DECK) + steam->sensor_update_rate_us = 4000; + else + steam->sensor_update_rate_us = 9000; INIT_WORK(&steam->unregister_work, steam_work_unregister_cb); /* @@ -1354,13 +1504,45 @@ static void steam_do_connect_event(struct steam_device *steam, bool connected) * Clamp the values to 32767..-32767 so that the range is * symmetrical and can be negated safely. */ -static inline s16 steam_le16(u8 *data) +static inline s16 steam_le16(const u8 *data) { - s16 x = (s16) le16_to_cpup((__le16 *)data); + s16 x = (s16) get_unaligned_le16((const __le16 *)data); return x == -32768 ? -32767 : x; } +struct steam_button_mapping { + int code; + u8 byte; + u8 bit; +}; + +struct steam_axis_mapping { + int code; + s8 sign; + u8 byte; +}; + +static void steam_map_buttons(struct input_dev *input, + const struct steam_button_mapping *mappings, const u8 *data) +{ + const struct steam_button_mapping *mapping; + + for (mapping = mappings; mapping->code; mapping++) + input_report_key(input, mapping->code, + data[mapping->byte] & BIT(mapping->bit)); +} + +static void steam_map_axes(struct input_dev *input, + const struct steam_axis_mapping *mappings, const u8 *data) +{ + const struct steam_axis_mapping *mapping; + + for (mapping = mappings; mapping->sign; mapping++) + input_report_abs(input, mapping->code, + mapping->sign * steam_le16(&data[mapping->byte])); +} + /* * The size for this message payload is 60. * The known values are: @@ -1427,18 +1609,52 @@ static inline s16 steam_le16(u8 *data) * 10.7 | -- | lpad_and_joy */ +static const struct steam_button_mapping steam_controller_button_mappings[] = { + { BTN_TR2, 8, 0 }, + { BTN_TL2, 8, 1 }, + { BTN_TR, 8, 2 }, + { BTN_TL, 8, 3 }, + { BTN_Y, 8, 4 }, + { BTN_B, 8, 5 }, + { BTN_X, 8, 6 }, + { BTN_A, 8, 7 }, + { BTN_SELECT, 9, 4 }, + { BTN_MODE, 9, 5 }, + { BTN_START, 9, 6 }, + { BTN_GRIPL, 9, 7 }, + { BTN_GRIPR, 10, 0 }, + { BTN_THUMBR, 10, 2 }, + { BTN_THUMBL, 10, 6 }, + { BTN_THUMB2, 10, 4 }, + { BTN_DPAD_UP, 9, 0 }, + { BTN_DPAD_RIGHT, 9, 1 }, + { BTN_DPAD_LEFT, 9, 2 }, + { BTN_DPAD_DOWN, 9, 3 }, + { /* sentinel */ }, +}; + +static const struct steam_axis_mapping steam_controller_axis_mappings[] = { + { ABS_RX, 1, 20 }, + { ABS_RY, -1, 22 }, + { /* sentinel */ }, +}; + +static const struct steam_axis_mapping steam_controller_imu_mappings[] = { + { ABS_X, 1, 28 }, + { ABS_Z, -1, 30 }, + { ABS_Y, 1, 32 }, + { ABS_RX, 1, 34 }, + { ABS_RZ, 1, 36 }, + { ABS_RY, 1, 38 }, + { /* sentinel */ }, +}; + static void steam_do_input_event(struct steam_device *steam, struct input_dev *input, u8 *data) { - /* 24 bits of buttons */ - u8 b8, b9, b10; s16 x, y; bool lpad_touched, lpad_and_joy; - b8 = data[8]; - b9 = data[9]; - b10 = data[10]; - input_report_abs(input, ABS_HAT2Y, data[11]); input_report_abs(input, ABS_HAT2X, data[12]); @@ -1450,8 +1666,8 @@ static void steam_do_input_event(struct steam_device *steam, * joystick values. * (lpad_touched || lpad_and_joy) tells if the lpad is really touched. */ - lpad_touched = b10 & BIT(3); - lpad_and_joy = b10 & BIT(7); + lpad_touched = data[10] & BIT(3); + lpad_and_joy = data[10] & BIT(7); x = steam_le16(data + 16); y = -steam_le16(data + 18); @@ -1467,35 +1683,25 @@ static void steam_do_input_event(struct steam_device *steam, input_report_abs(input, ABS_HAT0X, 0); input_report_abs(input, ABS_HAT0Y, 0); } + input_report_key(input, BTN_THUMB, lpad_touched || lpad_and_joy); - input_report_abs(input, ABS_RX, steam_le16(data + 20)); - input_report_abs(input, ABS_RY, -steam_le16(data + 22)); - - input_event(input, EV_KEY, BTN_TR2, !!(b8 & BIT(0))); - input_event(input, EV_KEY, BTN_TL2, !!(b8 & BIT(1))); - input_event(input, EV_KEY, BTN_TR, !!(b8 & BIT(2))); - input_event(input, EV_KEY, BTN_TL, !!(b8 & BIT(3))); - input_event(input, EV_KEY, BTN_Y, !!(b8 & BIT(4))); - input_event(input, EV_KEY, BTN_B, !!(b8 & BIT(5))); - input_event(input, EV_KEY, BTN_X, !!(b8 & BIT(6))); - input_event(input, EV_KEY, BTN_A, !!(b8 & BIT(7))); - input_event(input, EV_KEY, BTN_SELECT, !!(b9 & BIT(4))); - input_event(input, EV_KEY, BTN_MODE, !!(b9 & BIT(5))); - input_event(input, EV_KEY, BTN_START, !!(b9 & BIT(6))); - input_event(input, EV_KEY, BTN_GRIPL, !!(b9 & BIT(7))); - input_event(input, EV_KEY, BTN_GRIPR, !!(b10 & BIT(0))); - input_event(input, EV_KEY, BTN_THUMBR, !!(b10 & BIT(2))); - input_event(input, EV_KEY, BTN_THUMBL, !!(b10 & BIT(6))); - input_event(input, EV_KEY, BTN_THUMB, lpad_touched || lpad_and_joy); - input_event(input, EV_KEY, BTN_THUMB2, !!(b10 & BIT(4))); - input_event(input, EV_KEY, BTN_DPAD_UP, !!(b9 & BIT(0))); - input_event(input, EV_KEY, BTN_DPAD_RIGHT, !!(b9 & BIT(1))); - input_event(input, EV_KEY, BTN_DPAD_LEFT, !!(b9 & BIT(2))); - input_event(input, EV_KEY, BTN_DPAD_DOWN, !!(b9 & BIT(3))); + steam_map_buttons(input, steam_controller_button_mappings, data); + steam_map_axes(input, steam_controller_axis_mappings, data); input_sync(input); } +static void steam_do_sensors_event(struct steam_device *steam, + struct input_dev *sensors, u8 *data) +{ + steam->sensor_timestamp_us += steam->sensor_update_rate_us; + + input_event(sensors, EV_MSC, MSC_TIMESTAMP, steam->sensor_timestamp_us); + steam_map_axes(sensors, steam_controller_imu_mappings, data); + + input_sync(sensors); +} + /* * The size for this message payload is 56. * The known values are: @@ -1594,23 +1800,68 @@ static void steam_do_input_event(struct steam_device *steam, * 15.6 | -- | unknown * 15.7 | -- | unknown */ + +static const struct steam_button_mapping steam_deck_button_mappings[] = { + { BTN_TR2, 8, 0 }, + { BTN_TL2, 8, 1 }, + { BTN_TR, 8, 2 }, + { BTN_TL, 8, 3 }, + { BTN_Y, 8, 4 }, + { BTN_B, 8, 5 }, + { BTN_X, 8, 6 }, + { BTN_A, 8, 7 }, + { BTN_SELECT, 9, 4 }, + { BTN_MODE, 9, 5 }, + { BTN_START, 9, 6 }, + { BTN_GRIPL2, 9, 7 }, + { BTN_GRIPR2, 10, 0 }, + { BTN_THUMBL, 10, 6 }, + { BTN_THUMBR, 11, 2 }, + { BTN_DPAD_UP, 9, 0 }, + { BTN_DPAD_RIGHT, 9, 1 }, + { BTN_DPAD_LEFT, 9, 2 }, + { BTN_DPAD_DOWN, 9, 3 }, + { BTN_THUMB, 10, 1 }, + { BTN_THUMB2, 10, 2 }, + { BTN_GRIPL, 13, 1 }, + { BTN_GRIPR, 13, 2 }, + { BTN_BASE, 14, 2 }, + { /* sentinel */ }, +}; + +static const struct steam_axis_mapping steam_deck_axis_mappings[] = { + { ABS_X, 1, 48 }, + { ABS_Y, -1, 50 }, + { ABS_RX, 1, 52 }, + { ABS_RY, -1, 54 }, + { ABS_HAT2Y, 1, 44 }, + { ABS_HAT2X, 1, 46 }, + { /* sentinel */ }, +}; + +static const struct steam_axis_mapping steam_deck_imu_mappings[] = { + { ABS_X, 1, 24 }, + { ABS_Z, -1, 26 }, + { ABS_Y, 1, 28 }, + { ABS_RX, 1, 30 }, + { ABS_RZ, -1, 32 }, + { ABS_RY, 1, 34 }, + { /* sentinel */ }, +}; + static void steam_do_deck_input_event(struct steam_device *steam, struct input_dev *input, u8 *data) { - u8 b8, b9, b10, b11, b13, b14; + bool start_pressed; bool lpad_touched, rpad_touched; - b8 = data[8]; - b9 = data[9]; - b10 = data[10]; - b11 = data[11]; - b13 = data[13]; - b14 = data[14]; + start_pressed = data[9] & BIT(6); - if (!(b9 & BIT(6)) && steam->did_mode_switch) { + if (!start_pressed && steam->did_mode_switch) { steam->did_mode_switch = false; cancel_delayed_work(&steam->mode_switch); - } else if (!steam->client_opened && (b9 & BIT(6)) && !steam->did_mode_switch) { + } else if (!steam->client_opened && start_pressed && !steam->did_mode_switch) { + hid_dbg(steam->hdev, "%s: doing mode switch\n", __func__); steam->did_mode_switch = true; schedule_delayed_work(&steam->mode_switch, 45 * HZ / 100); } @@ -1618,8 +1869,8 @@ static void steam_do_deck_input_event(struct steam_device *steam, if (!steam->gamepad_mode && lizard_mode) return; - lpad_touched = b10 & BIT(3); - rpad_touched = b10 & BIT(4); + lpad_touched = data[10] & BIT(3); + rpad_touched = data[10] & BIT(4); if (lpad_touched) { input_report_abs(input, ABS_HAT0X, steam_le16(data + 16)); @@ -1637,38 +1888,8 @@ static void steam_do_deck_input_event(struct steam_device *steam, input_report_abs(input, ABS_HAT1Y, 0); } - input_report_abs(input, ABS_X, steam_le16(data + 48)); - input_report_abs(input, ABS_Y, -steam_le16(data + 50)); - input_report_abs(input, ABS_RX, steam_le16(data + 52)); - input_report_abs(input, ABS_RY, -steam_le16(data + 54)); - - input_report_abs(input, ABS_HAT2Y, steam_le16(data + 44)); - input_report_abs(input, ABS_HAT2X, steam_le16(data + 46)); - - input_event(input, EV_KEY, BTN_TR2, !!(b8 & BIT(0))); - input_event(input, EV_KEY, BTN_TL2, !!(b8 & BIT(1))); - input_event(input, EV_KEY, BTN_TR, !!(b8 & BIT(2))); - input_event(input, EV_KEY, BTN_TL, !!(b8 & BIT(3))); - input_event(input, EV_KEY, BTN_Y, !!(b8 & BIT(4))); - input_event(input, EV_KEY, BTN_B, !!(b8 & BIT(5))); - input_event(input, EV_KEY, BTN_X, !!(b8 & BIT(6))); - input_event(input, EV_KEY, BTN_A, !!(b8 & BIT(7))); - input_event(input, EV_KEY, BTN_SELECT, !!(b9 & BIT(4))); - input_event(input, EV_KEY, BTN_MODE, !!(b9 & BIT(5))); - input_event(input, EV_KEY, BTN_START, !!(b9 & BIT(6))); - input_event(input, EV_KEY, BTN_GRIPL2, !!(b9 & BIT(7))); - input_event(input, EV_KEY, BTN_GRIPR2, !!(b10 & BIT(0))); - input_event(input, EV_KEY, BTN_THUMBL, !!(b10 & BIT(6))); - input_event(input, EV_KEY, BTN_THUMBR, !!(b11 & BIT(2))); - input_event(input, EV_KEY, BTN_DPAD_UP, !!(b9 & BIT(0))); - input_event(input, EV_KEY, BTN_DPAD_RIGHT, !!(b9 & BIT(1))); - input_event(input, EV_KEY, BTN_DPAD_LEFT, !!(b9 & BIT(2))); - input_event(input, EV_KEY, BTN_DPAD_DOWN, !!(b9 & BIT(3))); - input_event(input, EV_KEY, BTN_THUMB, !!(b10 & BIT(1))); - input_event(input, EV_KEY, BTN_THUMB2, !!(b10 & BIT(2))); - input_event(input, EV_KEY, BTN_GRIPL, !!(b13 & BIT(1))); - input_event(input, EV_KEY, BTN_GRIPR, !!(b13 & BIT(2))); - input_event(input, EV_KEY, BTN_BASE, !!(b14 & BIT(2))); + steam_map_buttons(input, steam_deck_button_mappings, data); + steam_map_axes(input, steam_deck_axis_mappings, data); input_sync(input); } @@ -1676,25 +1897,13 @@ static void steam_do_deck_input_event(struct steam_device *steam, static void steam_do_deck_sensors_event(struct steam_device *steam, struct input_dev *sensors, u8 *data) { - /* - * The deck input report is received every 4 ms on average, - * with a jitter of +/- 4 ms even though the USB descriptor claims - * that it uses 1 kHz. - * Since the HID report does not include a sensor timestamp, - * use a fixed increment here. - */ - steam->sensor_timestamp_us += 4000; + steam->sensor_timestamp_us += steam->sensor_update_rate_us; if (!steam->gamepad_mode && lizard_mode) return; input_event(sensors, EV_MSC, MSC_TIMESTAMP, steam->sensor_timestamp_us); - input_report_abs(sensors, ABS_X, steam_le16(data + 24)); - input_report_abs(sensors, ABS_Z, -steam_le16(data + 26)); - input_report_abs(sensors, ABS_Y, steam_le16(data + 28)); - input_report_abs(sensors, ABS_RX, steam_le16(data + 30)); - input_report_abs(sensors, ABS_RZ, -steam_le16(data + 32)); - input_report_abs(sensors, ABS_RY, steam_le16(data + 34)); + steam_map_axes(sensors, steam_deck_imu_mappings, data); input_sync(sensors); } @@ -1773,6 +1982,9 @@ static int steam_raw_event(struct hid_device *hdev, input = rcu_dereference(steam->input); if (likely(input)) steam_do_input_event(steam, input, data); + sensors = rcu_dereference(steam->sensors); + if (likely(sensors)) + steam_do_sensors_event(steam, sensors, data); rcu_read_unlock(); break; case ID_CONTROLLER_DECK_STATE: diff --git a/drivers/hid/hid-tmff.c b/drivers/hid/hid-tmff.c index 423f395d01ac5f..319e7b6703846a 100644 --- a/drivers/hid/hid-tmff.c +++ b/drivers/hid/hid-tmff.c @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -47,9 +48,9 @@ struct tmff_device { /* Changes values from 0 to 0xffff into values from minimum to maximum */ static inline int tmff_scale_u16(unsigned int in, int minimum, int maximum) { - int ret; + s64 ret; - ret = (in * (maximum - minimum) / 0xffff) + minimum; + ret = div_s64((s64)in * ((s64)maximum - minimum), 0xffff) + minimum; if (ret < minimum) return minimum; if (ret > maximum) @@ -60,9 +61,9 @@ static inline int tmff_scale_u16(unsigned int in, int minimum, int maximum) /* Changes values from -0x80 to 0x7f into values from minimum to maximum */ static inline int tmff_scale_s8(int in, int minimum, int maximum) { - int ret; + s64 ret; - ret = (((in + 0x80) * (maximum - minimum)) / 0xff) + minimum; + ret = div_s64((s64)(in + 0x80) * ((s64)maximum - minimum), 0xff) + minimum; if (ret < minimum) return minimum; if (ret > maximum) diff --git a/drivers/hid/i2c-hid/Makefile b/drivers/hid/i2c-hid/Makefile index 55bd5e0f35af32..38d5d827f3ce4f 100644 --- a/drivers/hid/i2c-hid/Makefile +++ b/drivers/hid/i2c-hid/Makefile @@ -8,7 +8,7 @@ obj-$(CONFIG_I2C_HID_CORE) += i2c-hid.o i2c-hid-objs = i2c-hid-core.o i2c-hid-$(CONFIG_DMI) += i2c-hid-dmi-quirks.o -obj-$(CONFIG_I2C_HID_ACPI) += i2c-hid-acpi.o +obj-$(CONFIG_I2C_HID_ACPI) += i2c-hid-acpi.o i2c-hid-acpi-prp0001.o obj-$(CONFIG_I2C_HID_OF) += i2c-hid-of.o obj-$(CONFIG_I2C_HID_OF_ELAN) += i2c-hid-of-elan.o obj-$(CONFIG_I2C_HID_OF_GOODIX) += i2c-hid-of-goodix.o diff --git a/drivers/hid/i2c-hid/i2c-hid-acpi-prp0001.c b/drivers/hid/i2c-hid/i2c-hid-acpi-prp0001.c new file mode 100644 index 00000000000000..d2cf4714ae7f1c --- /dev/null +++ b/drivers/hid/i2c-hid/i2c-hid-acpi-prp0001.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * HID over I2C driver for PRP0001 devices missing hid-descr-addr + * + * Some devices, for example the Lenovo KaiTian N60d and Inspur CP300L3, use + * _HID "PRP0001" with _DSD compatible "hid-over-i2c" but lack "hid-descr-addr" + * from the _DSD. The HID descriptor address is provided only through an ACPI + * _DSM. The TPD0 node in the DSDT shows _DSM Function 1 returning 0x20. + * + * Copyright (C) 2026 谢致邦 (XIE Zhibang) + */ + +#include +#include +#include +#include +#include + +#include "i2c-hid.h" +#include "i2c-hid-acpi.h" + +static int i2c_hid_acpi_prp0001_power_up(struct i2chid_ops *ops) +{ + /* give the device time to power up */ + msleep(750); + return 0; +} + +static struct i2chid_ops i2c_hid_acpi_prp0001_ops = { + .power_up = i2c_hid_acpi_prp0001_power_up, + /* + * No .restore_sequence needed: the _DSM on these devices returns a + * constant (0x20) with no side effects, unlike some PNP0C50 _DSM + * implementations that switch the hardware between PS/2 and I2C modes. + */ +}; + +static int i2c_hid_acpi_prp0001_probe(struct i2c_client *client) +{ + struct device *dev = &client->dev; + struct acpi_device *adev; + u16 hid_descriptor_address; + int ret; + + /* If hid-descr-addr is present, let i2c-hid-of handle it */ + if (device_property_present(dev, "hid-descr-addr")) + return -ENODEV; + + adev = ACPI_COMPANION(dev); + if (!adev) + return -ENODEV; + + ret = i2c_hid_acpi_get_descriptor(adev); + if (ret < 0) + return ret; + dev_warn(dev, + "hid-descr-addr device property NOT found, using ACPI _DSM fallback. Contact vendor for firmware update!\n"); + hid_descriptor_address = ret; + + /* + * No acpi_device_fix_up_power() needed: TPD0 has no _PS0, _PS3, _PSC + * or _PRx methods and follows I2C bus power. + */ + return i2c_hid_core_probe(client, &i2c_hid_acpi_prp0001_ops, + hid_descriptor_address, 0); +} + +static const struct of_device_id i2c_hid_acpi_prp0001_of_match[] = { + { .compatible = "hid-over-i2c" }, + {}, +}; +MODULE_DEVICE_TABLE(of, i2c_hid_acpi_prp0001_of_match); + +static const struct i2c_device_id i2c_hid_acpi_prp0001_id[] = { + { .name = "hid-over-i2c" }, + { } +}; +MODULE_DEVICE_TABLE(i2c, i2c_hid_acpi_prp0001_id); + +static struct i2c_driver i2c_hid_acpi_prp0001_driver = { + .driver = { + .name = "i2c_hid_acpi_prp0001", + .pm = &i2c_hid_core_pm, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + /* + * of_match_ptr() makes this NULL when CONFIG_OF=n, but that's + * fine: the I2C id_table with "hid-over-i2c" handles matching + * via client->name (set by acpi_set_modalias() from the _DSD + * compatible property). + */ + .of_match_table = of_match_ptr(i2c_hid_acpi_prp0001_of_match), + }, + + .probe = i2c_hid_acpi_prp0001_probe, + .remove = i2c_hid_core_remove, + .shutdown = i2c_hid_core_shutdown, + .id_table = i2c_hid_acpi_prp0001_id, +}; + +module_i2c_driver(i2c_hid_acpi_prp0001_driver); + +MODULE_DESCRIPTION("HID over I2C driver for PRP0001 devices missing hid-descr-addr"); +MODULE_AUTHOR("谢致邦 (XIE Zhibang) "); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/i2c-hid/i2c-hid-acpi.c b/drivers/hid/i2c-hid/i2c-hid-acpi.c index abd700a101f46c..13f977d6aab615 100644 --- a/drivers/hid/i2c-hid/i2c-hid-acpi.c +++ b/drivers/hid/i2c-hid/i2c-hid-acpi.c @@ -25,9 +25,9 @@ #include #include #include -#include #include "i2c-hid.h" +#include "i2c-hid-acpi.h" struct i2c_hid_acpi { struct i2chid_ops ops; @@ -48,39 +48,11 @@ static const struct acpi_device_id i2c_hid_acpi_blacklist[] = { { } }; -/* HID I²C Device: 3cdff6f7-4267-4555-ad05-b30a3d8938de */ -static guid_t i2c_hid_guid = - GUID_INIT(0x3CDFF6F7, 0x4267, 0x4555, - 0xAD, 0x05, 0xB3, 0x0A, 0x3D, 0x89, 0x38, 0xDE); - -static int i2c_hid_acpi_get_descriptor(struct i2c_hid_acpi *ihid_acpi) -{ - struct acpi_device *adev = ihid_acpi->adev; - acpi_handle handle = acpi_device_handle(adev); - union acpi_object *obj; - u16 hid_descriptor_address; - - if (acpi_match_device_ids(adev, i2c_hid_acpi_blacklist) == 0) - return -ENODEV; - - obj = acpi_evaluate_dsm_typed(handle, &i2c_hid_guid, 1, 1, NULL, - ACPI_TYPE_INTEGER); - if (!obj) { - acpi_handle_err(handle, "Error _DSM call to get HID descriptor address failed\n"); - return -ENODEV; - } - - hid_descriptor_address = obj->integer.value; - ACPI_FREE(obj); - - return hid_descriptor_address; -} - static void i2c_hid_acpi_restore_sequence(struct i2chid_ops *ops) { struct i2c_hid_acpi *ihid_acpi = container_of(ops, struct i2c_hid_acpi, ops); - i2c_hid_acpi_get_descriptor(ihid_acpi); + i2c_hid_acpi_get_descriptor(ihid_acpi->adev); } static void i2c_hid_acpi_shutdown_tail(struct i2chid_ops *ops) @@ -93,24 +65,28 @@ static void i2c_hid_acpi_shutdown_tail(struct i2chid_ops *ops) static int i2c_hid_acpi_probe(struct i2c_client *client) { struct device *dev = &client->dev; + struct acpi_device *adev = ACPI_COMPANION(dev); struct i2c_hid_acpi *ihid_acpi; u16 hid_descriptor_address; int ret; - ihid_acpi = devm_kzalloc(&client->dev, sizeof(*ihid_acpi), GFP_KERNEL); + if (acpi_match_device_ids(adev, i2c_hid_acpi_blacklist) == 0) + return -ENODEV; + + ret = i2c_hid_acpi_get_descriptor(adev); + if (ret < 0) + return ret; + hid_descriptor_address = ret; + + ihid_acpi = devm_kzalloc(dev, sizeof(*ihid_acpi), GFP_KERNEL); if (!ihid_acpi) return -ENOMEM; - ihid_acpi->adev = ACPI_COMPANION(dev); + ihid_acpi->adev = adev; ihid_acpi->ops.shutdown_tail = i2c_hid_acpi_shutdown_tail; ihid_acpi->ops.restore_sequence = i2c_hid_acpi_restore_sequence; - ret = i2c_hid_acpi_get_descriptor(ihid_acpi); - if (ret < 0) - return ret; - hid_descriptor_address = ret; - - acpi_device_fix_up_power(ihid_acpi->adev); + acpi_device_fix_up_power(adev); return i2c_hid_core_probe(client, &ihid_acpi->ops, hid_descriptor_address, 0); diff --git a/drivers/hid/i2c-hid/i2c-hid-acpi.h b/drivers/hid/i2c-hid/i2c-hid-acpi.h new file mode 100644 index 00000000000000..0bbed1853313dc --- /dev/null +++ b/drivers/hid/i2c-hid/i2c-hid-acpi.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _I2C_HID_ACPI_H +#define _I2C_HID_ACPI_H + +#include +#include + +static inline int i2c_hid_acpi_get_descriptor(struct acpi_device *adev) +{ + /* HID I²C Device: 3cdff6f7-4267-4555-ad05-b30a3d8938de */ + static const guid_t i2c_hid_guid = + GUID_INIT(0x3CDFF6F7, 0x4267, 0x4555, + 0xAD, 0x05, 0xB3, 0x0A, 0x3D, 0x89, 0x38, 0xDE); + + acpi_handle handle = acpi_device_handle(adev); + union acpi_object *obj; + u16 addr; + + obj = acpi_evaluate_dsm_typed(handle, &i2c_hid_guid, + 1, 1, NULL, ACPI_TYPE_INTEGER); + if (!obj) { + acpi_handle_err(handle, + "Error _DSM call to get HID descriptor address failed\n"); + return -ENODEV; + } + + addr = obj->integer.value; + ACPI_FREE(obj); + return addr; +} + +#endif diff --git a/drivers/hid/i2c-hid/i2c-hid-core.c b/drivers/hid/i2c-hid/i2c-hid-core.c index 3adb16366e9394..0e725a0f0abee9 100644 --- a/drivers/hid/i2c-hid/i2c-hid-core.c +++ b/drivers/hid/i2c-hid/i2c-hid-core.c @@ -792,7 +792,7 @@ static int i2c_hid_parse(struct hid_device *hid) ihid->hdesc.wReportDescRegister, rdesc, rsize); if (ret) { - hid_err(hid, "reading report descriptor failed\n"); + dev_err(&client->dev, "reading report descriptor failed\n"); goto out; } } diff --git a/drivers/hid/i2c-hid/i2c-hid-of-goodix.c b/drivers/hid/i2c-hid/i2c-hid-of-goodix.c index f1597ad67e7c8a..f4dbcd1d1d4729 100644 --- a/drivers/hid/i2c-hid/i2c-hid-of-goodix.c +++ b/drivers/hid/i2c-hid/i2c-hid-of-goodix.c @@ -51,8 +51,10 @@ static int goodix_i2c_hid_power_up(struct i2chid_ops *ops) return ret; ret = regulator_enable(ihid_goodix->vddio); - if (ret) + if (ret) { + regulator_disable(ihid_goodix->vdd); return ret; + } if (ihid_goodix->timings->post_power_delay_ms) msleep(ihid_goodix->timings->post_power_delay_ms); diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 6824bd7cb3c4fd..18ee549d988095 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -2982,6 +2982,13 @@ static int __init hv_acpi_init(void) return -ENODEV; if (hv_root_partition() && !hv_nested) + /* + * A non-nested root partition does not need VMBus client + * functionality. However, the mshv_root module may have + * a dependency on the VMBus module as described in + * commit 840b740a35bf. Return success so the module + * loads even though no VMBus initialization is done. + */ return 0; /* @@ -3030,6 +3037,14 @@ static void __exit vmbus_exit(void) { int cpu; + if (hv_root_partition() && !hv_nested) + /* + * If a non-nested root partition loaded the VMBus module, + * hv_acpi_init() did not do any VMBus initialization. + * There's nothing to clean up, so just return. + */ + return; + unregister_syscore(&hv_synic_syscore); hv_remove_kexec_handler(); diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c index 6215ea49faaa9e..ab9c8cbf887aa7 100644 --- a/drivers/hwmon/coretemp.c +++ b/drivers/hwmon/coretemp.c @@ -650,6 +650,7 @@ static void coretemp_device_remove(int zoneid) struct platform_data *pdata = platform_get_drvdata(pdev); ida_destroy(&pdata->ida); + kfree(pdata->core_data); kfree(pdata); platform_device_unregister(pdev); } diff --git a/drivers/hwmon/cros_ec_hwmon.c b/drivers/hwmon/cros_ec_hwmon.c index 03bfcc40bb7c3e..b65518fa7c9cc8 100644 --- a/drivers/hwmon/cros_ec_hwmon.c +++ b/drivers/hwmon/cros_ec_hwmon.c @@ -5,6 +5,7 @@ * Copyright (C) 2024 Thomas Weißschuh */ +#include #include #include #include @@ -24,6 +25,7 @@ struct cros_ec_hwmon_priv { struct cros_ec_device *cros_ec; + struct device *hwmon_dev; const char *temp_sensor_names[EC_TEMP_SENSOR_ENTRIES + EC_TEMP_SENSOR_B_ENTRIES]; u8 usable_fans; bool fan_control_supported; @@ -401,6 +403,8 @@ static int cros_ec_hwmon_cooling_get_cur_state(struct thermal_cooling_device *cd u8 read_val; int ret; + guard(hwmon_lock)(priv->hwmon_priv->hwmon_dev); + ret = cros_ec_hwmon_read_pwm_value(priv->hwmon_priv->cros_ec, priv->index, &read_val); if (ret) return ret; @@ -414,6 +418,8 @@ static int cros_ec_hwmon_cooling_set_cur_state(struct thermal_cooling_device *cd { const struct cros_ec_hwmon_cooling_priv *priv = cdev->devdata; + guard(hwmon_lock)(priv->hwmon_priv->hwmon_dev); + return cros_ec_hwmon_write_pwm_input(priv->hwmon_priv->cros_ec, priv->index, val); } @@ -547,7 +553,6 @@ static int cros_ec_hwmon_probe(struct platform_device *pdev) struct cros_ec_dev *ec_dev = dev_get_drvdata(dev->parent); struct cros_ec_device *cros_ec = ec_dev->ec_dev; struct cros_ec_hwmon_priv *priv; - struct device *hwmon_dev; u8 thermal_version; int ret; @@ -570,13 +575,17 @@ static int cros_ec_hwmon_probe(struct platform_device *pdev) priv->fan_control_supported = cros_ec_hwmon_probe_fan_control_supported(priv->cros_ec); priv->temp_threshold_supported = is_cros_ec_cmd_available(priv->cros_ec, EC_CMD_THERMAL_GET_THRESHOLD, 1); + + priv->hwmon_dev = devm_hwmon_device_register_with_info(dev, "cros_ec", priv, + &cros_ec_hwmon_chip_info, NULL); + if (IS_ERR(priv->hwmon_dev)) + return PTR_ERR(priv->hwmon_dev); + cros_ec_hwmon_register_fan_cooling_devices(dev, priv); - hwmon_dev = devm_hwmon_device_register_with_info(dev, "cros_ec", priv, - &cros_ec_hwmon_chip_info, NULL); platform_set_drvdata(pdev, priv); - return PTR_ERR_OR_ZERO(hwmon_dev); + return 0; } static int cros_ec_hwmon_suspend(struct platform_device *pdev, pm_message_t state) diff --git a/drivers/hwmon/emc1403.c b/drivers/hwmon/emc1403.c index cd753b38709f25..2f6dadbce68a5a 100644 --- a/drivers/hwmon/emc1403.c +++ b/drivers/hwmon/emc1403.c @@ -305,10 +305,9 @@ static int emc1403_get_hyst(struct thermal_data *data, int channel, ret = regmap_read(data->regmap, 0x21, &hyst); if (ret < 0) return ret; - if (map == temp_min) - *val = limit + hyst * 1000; - else - *val = limit - hyst * 1000; + + *val = limit - hyst * 1000; + return 0; } @@ -324,9 +323,6 @@ static int emc1403_temp_read(struct thermal_data *data, u32 attr, int channel, l case hwmon_temp_input: ret = emc1403_get_temp(data, channel, ema1403_temp_map[attr], val); break; - case hwmon_temp_min_hyst: - ret = emc1403_get_hyst(data, channel, temp_min, val); - break; case hwmon_temp_max_hyst: ret = emc1403_get_hyst(data, channel, temp_max, val); break; @@ -548,7 +544,6 @@ static umode_t emc1403_temp_is_visible(const void *_data, u32 attr, int channel) case hwmon_temp_max_alarm: case hwmon_temp_crit_alarm: case hwmon_temp_fault: - case hwmon_temp_min_hyst: case hwmon_temp_max_hyst: return 0444; case hwmon_temp_min: @@ -591,35 +586,35 @@ static const struct hwmon_channel_info * const emc1403_info[] = { HWMON_CHANNEL_INFO(chip, HWMON_C_UPDATE_INTERVAL), HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT ), diff --git a/drivers/hwmon/ltc4283.c b/drivers/hwmon/ltc4283.c index 9b85293ea66498..9ec409bc264822 100644 --- a/drivers/hwmon/ltc4283.c +++ b/drivers/hwmon/ltc4283.c @@ -1772,6 +1772,7 @@ static const struct of_device_id ltc4283_of_match[] = { { .compatible = "adi,ltc4283" }, { } }; +MODULE_DEVICE_TABLE(of, ltc4283_of_match); static const struct i2c_device_id ltc4283_i2c_id[] = { { "ltc4283" }, diff --git a/drivers/hwspinlock/hwspinlock_core.c b/drivers/hwspinlock/hwspinlock_core.c index cc8e952a67727b..a509b73da190d3 100644 --- a/drivers/hwspinlock/hwspinlock_core.c +++ b/drivers/hwspinlock/hwspinlock_core.c @@ -472,7 +472,7 @@ static int hwspin_lock_register_single(struct hwspinlock *hwlock, int id) out: mutex_unlock(&hwspinlock_tree_lock); - return 0; + return ret; } static struct hwspinlock *hwspin_lock_unregister_single(unsigned int id) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-cfg.c b/drivers/hwtracing/coresight/coresight-etm4x-cfg.c index c302072b293a3b..e1a59b4345052c 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-cfg.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-cfg.c @@ -76,7 +76,7 @@ static int etm4_cfg_map_reg_offset(struct etmv4_drvdata *drvdata, } else if ((offset & GENMASK(11, 4)) == TRCSEQEVRn(0)) { /* sequencer state control registers */ idx = (offset & GENMASK(3, 0)) / 4; - if (idx < ETM_MAX_SEQ_STATES) { + if (idx < ETM_MAX_SEQ_TRANSITIONS) { reg_csdev->driver_regval = &drvcfg->seq_ctrl[idx]; err = 0; } diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 14bb31bd6a0b97..2247ad55d44425 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -93,7 +93,7 @@ static int etm4_probe_cpu(unsigned int cpu); static bool etm4x_sspcicrn_present(struct etmv4_drvdata *drvdata, int n) { return (n < drvdata->nr_ss_cmp) && - drvdata->nr_pe && + drvdata->nr_pe_cmp && (drvdata->config.ss_status[n] & TRCSSCSRn_PC); } @@ -542,7 +542,8 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) etm4x_relaxed_write32(csa, config->vissctlr, TRCVISSCTLR); if (drvdata->nr_pe_cmp) etm4x_relaxed_write32(csa, config->vipcssctlr, TRCVIPCSSCTLR); - for (i = 0; i < drvdata->nrseqstate - 1; i++) + + for (i = 0; i < drvdata->nr_seq_ctrls; i++) etm4x_relaxed_write32(csa, config->seq_ctrl[i], TRCSEQEVRn(i)); if (drvdata->nrseqstate) { etm4x_relaxed_write32(csa, config->seq_rst, TRCSEQRSTEVR); @@ -756,8 +757,7 @@ static int etm4_parse_event_config(struct coresight_device *csdev, .ATTR_CFG_FLD_timestamp_CFG = U64_MAX, }; struct perf_event_attr *attr = &event->attr; - unsigned long cfg_hash; - int preset, cc_threshold; + int cc_threshold; u8 ts_level; /* Clear configuration from previous run */ @@ -843,16 +843,6 @@ static int etm4_parse_event_config(struct coresight_device *csdev, /* bit[12], Return stack enable bit */ config->cfg |= TRCCONFIGR_RS; - /* - * Set any selected configuration and preset. A zero configid means no - * configuration active, preset = 0 means no preset selected. - */ - cfg_hash = ATTR_CFG_GET_FLD(attr, configid); - if (cfg_hash) { - preset = ATTR_CFG_GET_FLD(attr, preset); - ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset); - } - /* branch broadcast - enable if selected and supported */ if (ATTR_CFG_GET_FLD(attr, branch_broadcast)) { if (!drvdata->trcbb) { @@ -876,7 +866,9 @@ static int etm4_enable_perf(struct coresight_device *csdev, struct coresight_path *path) { struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - int ret; + struct perf_event_attr *attr = &event->attr; + unsigned long cfg_hash; + int ret, preset; if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) return -EINVAL; @@ -887,7 +879,19 @@ static int etm4_enable_perf(struct coresight_device *csdev, /* Configure the tracer based on the session's specifics */ ret = etm4_parse_event_config(csdev, event); if (ret) - goto out; + goto err; + + /* + * Set any selected configuration and preset. A zero configid means no + * configuration active, preset = 0 means no preset selected. + */ + cfg_hash = ATTR_CFG_GET_FLD(attr, configid); + if (cfg_hash) { + preset = ATTR_CFG_GET_FLD(attr, preset); + ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset); + if (ret) + goto err; + } drvdata->trcid = path->trace_id; @@ -896,16 +900,19 @@ static int etm4_enable_perf(struct coresight_device *csdev, /* And enable it */ ret = etm4_enable_hw(drvdata); - -out: - /* Failed to start tracer; roll back to DISABLED mode */ if (ret) { - coresight_set_mode(csdev, CS_MODE_DISABLED); - return ret; + if (cfg_hash) + cscfg_csdev_disable_active_config(csdev); + goto err; } csdev->path = path; return 0; + +err: + /* Failed to start tracer; roll back to DISABLED mode */ + coresight_set_mode(csdev, CS_MODE_DISABLED); + return ret; } static int etm4_enable_sysfs(struct coresight_device *csdev, struct coresight_path *path) @@ -919,8 +926,10 @@ static int etm4_enable_sysfs(struct coresight_device *csdev, struct coresight_pa cscfg_config_sysfs_get_active_cfg(&cfg_hash, &preset); if (cfg_hash) { ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset); - if (ret) + if (ret) { + etm4_release_trace_id(drvdata); return ret; + } } raw_spin_lock(&drvdata->spinlock); @@ -1508,6 +1517,8 @@ static void etm4_init_arch_data(void *info) drvdata->lpoverride = (etmidr5 & TRCIDR5_LPOVERRIDE) && (!drvdata->skip_power_up); /* NUMSEQSTATE, bits[27:25] number of sequencer states implemented */ drvdata->nrseqstate = FIELD_GET(TRCIDR5_NUMSEQSTATE_MASK, etmidr5); + if (drvdata->nrseqstate) + drvdata->nr_seq_ctrls = ETM_MAX_SEQ_TRANSITIONS; /* NUMCNTR, bits[30:28] number of counters available for tracing */ drvdata->nr_cntr = FIELD_GET(TRCIDR5_NUMCNTR_MASK, etmidr5); @@ -1896,7 +1907,7 @@ static int etm4_cpu_save(struct coresight_device *csdev) if (drvdata->nr_pe_cmp) state->trcvipcssctlr = etm4x_read32(csa, TRCVIPCSSCTLR); - for (i = 0; i < drvdata->nrseqstate - 1; i++) + for (i = 0; i < drvdata->nr_seq_ctrls; i++) state->trcseqevr[i] = etm4x_read32(csa, TRCSEQEVRn(i)); if (drvdata->nrseqstate) { @@ -2009,7 +2020,7 @@ static void etm4_cpu_restore(struct coresight_device *csdev) if (drvdata->nr_pe_cmp) etm4x_relaxed_write32(csa, state->trcvipcssctlr, TRCVIPCSSCTLR); - for (i = 0; i < drvdata->nrseqstate - 1; i++) + for (i = 0; i < drvdata->nr_seq_ctrls; i++) etm4x_relaxed_write32(csa, state->trcseqevr[i], TRCSEQEVRn(i)); if (drvdata->nrseqstate) { diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c index e9eeea6240d557..cc6cdd3ae29d50 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c @@ -223,7 +223,7 @@ static ssize_t reset_store(struct device *dev, config->vipcssctlr = 0x0; /* Disable seq events */ - for (i = 0; i < drvdata->nrseqstate-1; i++) + for (i = 0; i < drvdata->nr_seq_ctrls; i++) config->seq_ctrl[i] = 0x0; config->seq_rst = 0x0; config->seq_state = 0x0; @@ -1395,9 +1395,11 @@ static ssize_t seq_idx_store(struct device *dev, struct etmv4_drvdata *drvdata = dev_get_drvdata(dev->parent); struct etmv4_config *config = &drvdata->config; + if (!drvdata->nr_seq_ctrls) + return -ENOTSUPP; if (kstrtoul(buf, 16, &val)) return -EINVAL; - if (val >= drvdata->nrseqstate - 1) + if (val >= drvdata->nr_seq_ctrls) return -EINVAL; /* diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index 89d81ce4e04e2d..84db8b97c98afe 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -614,6 +614,7 @@ static inline u32 etm4_res_sel_pair(u8 res_sel_idx) #define ETM_MAX_NR_PE 8 #define ETMv4_MAX_CNTR 4 #define ETM_MAX_SEQ_STATES 4 +#define ETM_MAX_SEQ_TRANSITIONS 3 #define ETM_MAX_EXT_INP_SEL 4 #define ETM_MAX_EXT_INP 256 #define ETM_MAX_EXT_OUT 4 @@ -877,7 +878,7 @@ struct etmv4_config { u32 vipcssctlr; u8 seq_idx; u8 syncfreq; - u32 seq_ctrl[ETM_MAX_SEQ_STATES]; + u32 seq_ctrl[ETM_MAX_SEQ_TRANSITIONS]; u32 seq_rst; u32 seq_state; u8 cntr_idx; @@ -928,7 +929,7 @@ struct etmv4_save_state { u32 trcvissctlr; u32 trcvipcssctlr; - u32 trcseqevr[ETM_MAX_SEQ_STATES]; + u32 trcseqevr[ETM_MAX_SEQ_TRANSITIONS]; u32 trcseqrstevr; u32 trcseqstr; u32 trcextinselr; @@ -981,6 +982,7 @@ struct etmv4_save_state { * @numcidc: Number of contextID comparators. * @numvmidc: Number of VMID comparators. * @nrseqstate: The number of sequencer states that are implemented. + * @nr_seq_ctrls: The number of sequence state transition control registers. * @nr_event: Indicates how many events the trace unit support. * @nr_resource:The number of resource selection pairs available for tracing. * @nr_ss_cmp: Number of single-shot comparator controls that are available. @@ -1046,6 +1048,7 @@ struct etmv4_drvdata { u8 numextinsel; u8 numvmidc; u8 nrseqstate; + u8 nr_seq_ctrls; u8 nr_event; u8 nr_resource; u8 nr_ss_cmp; diff --git a/drivers/i2c/busses/i2c-ocores.c b/drivers/i2c/busses/i2c-ocores.c index df6ebf32d6e8f3..2d18c10358375f 100644 --- a/drivers/i2c/busses/i2c-ocores.c +++ b/drivers/i2c/busses/i2c-ocores.c @@ -755,7 +755,11 @@ static int ocores_i2c_resume(struct device *dev) rate = clk_get_rate(i2c->clk) / 1000; if (rate) i2c->ip_clock_khz = rate; - return ocores_init(dev, i2c); + ret = ocores_init(dev, i2c); + if (ret) + clk_disable_unprepare(i2c->clk); + + return ret; } static DEFINE_NOIRQ_DEV_PM_OPS(ocores_i2c_pm, diff --git a/drivers/i3c/master.c b/drivers/i3c/master.c index b51b159fadeed2..62b7a156291037 100644 --- a/drivers/i3c/master.c +++ b/drivers/i3c/master.c @@ -820,6 +820,11 @@ static struct attribute *i3c_masterdev_attrs[] = { }; ATTRIBUTE_GROUPS(i3c_masterdev); +static void i3c_master_free_i3c_dev(struct i3c_dev_desc *dev) +{ + kfree(dev); +} + static void i3c_masterdev_release(struct device *dev) { struct i3c_master_controller *master = dev_to_i3cmaster(dev); @@ -832,6 +837,8 @@ static void i3c_masterdev_release(struct device *dev) i3c_bus_cleanup(bus); of_node_put(dev->of_node); + + i3c_master_free_i3c_dev(master->this); } static const struct device_type i3c_masterdev_type = { @@ -1042,11 +1049,6 @@ static void i3c_device_release(struct device *dev) kfree(i3cdev); } -static void i3c_master_free_i3c_dev(struct i3c_dev_desc *dev) -{ - kfree(dev); -} - static struct i3c_dev_desc * i3c_master_alloc_i3c_dev(struct i3c_master_controller *master, const struct i3c_device_info *info) @@ -1898,12 +1900,21 @@ err_free_dev: static void i3c_master_register_new_i3c_devs(struct i3c_master_controller *master) { + struct i3c_device *i3cdev, *tmp; struct i3c_dev_desc *desc; + LIST_HEAD(i3c_unreg_devs); int ret; if (!master->init_done) return; + i3c_bus_maintenance_lock(&master->bus); + + if (master->shutting_down) { + i3c_bus_maintenance_unlock(&master->bus); + return; + } + i3c_bus_for_each_i3cdev(&master->bus, desc) { if (desc->dev || !desc->info.dyn_addr || desc == master->this) continue; @@ -1924,23 +1935,37 @@ i3c_master_register_new_i3c_devs(struct i3c_master_controller *master) if (desc->boardinfo) desc->dev->dev.of_node = desc->boardinfo->of_node; - ret = device_register(&desc->dev->dev); - if (ret) { - dev_err(&master->dev, - "Failed to add I3C device (err = %d)\n", ret); - put_device(&desc->dev->dev); - } + list_add_tail(&desc->dev->node, &i3c_unreg_devs); } + + i3c_bus_maintenance_unlock(&master->bus); + + list_for_each_entry_safe(i3cdev, tmp, &i3c_unreg_devs, node) { + ret = device_register(&i3cdev->dev); + if (ret) + dev_err(&master->dev, "Failed to add I3C device (err = %d)\n", ret); + else + list_del_init(&i3cdev->node); + } + + i3c_bus_maintenance_lock(&master->bus); + + list_for_each_entry_safe(i3cdev, tmp, &i3c_unreg_devs, node) { + list_del(&i3cdev->node); + desc = i3cdev->desc; + i3cdev->desc = NULL; + put_device(&i3cdev->dev); + desc->dev = NULL; + } + + i3c_bus_maintenance_unlock(&master->bus); } static void i3c_master_reg_work_fn(struct work_struct *work) { struct i3c_master_controller *master = container_of(work, typeof(*master), reg_work); - i3c_bus_normaluse_lock(&master->bus); - if (!master->shutting_down) - i3c_master_register_new_i3c_devs(master); - i3c_bus_normaluse_unlock(&master->bus); + i3c_master_register_new_i3c_devs(master); } /** @@ -2069,6 +2094,8 @@ int i3c_master_set_info(struct i3c_master_controller *master, return 0; err_free_dev: + master->bus.cur_master = NULL; + master->this = NULL; i3c_master_free_i3c_dev(i3cdev); return ret; @@ -2089,7 +2116,8 @@ static void i3c_master_detach_free_devs(struct i3c_master_controller *master) i3cdev->boardinfo->init_dyn_addr, I3C_ADDR_SLOT_FREE); - i3c_master_free_i3c_dev(i3cdev); + if (i3cdev != master->this) + i3c_master_free_i3c_dev(i3cdev); } list_for_each_entry_safe(i2cdev, i2ctmp, &master->bus.devs.i2c, @@ -2321,7 +2349,8 @@ i3c_master_search_i3c_dev_duplicate(struct i3c_dev_desc *refdev) struct i3c_dev_desc *i3cdev; i3c_bus_for_each_i3cdev(&master->bus, i3cdev) { - if (i3cdev != refdev && i3cdev->info.pid == refdev->info.pid) + if (i3cdev != refdev && i3cdev->info.pid == refdev->info.pid && + i3cdev != master->this) return i3cdev; } diff --git a/drivers/i3c/master/adi-i3c-master.c b/drivers/i3c/master/adi-i3c-master.c index e29aac28695775..fb4cfc9026ccf5 100644 --- a/drivers/i3c/master/adi-i3c-master.c +++ b/drivers/i3c/master/adi-i3c-master.c @@ -929,6 +929,7 @@ static const struct of_device_id adi_i3c_master_of_match[] = { { .compatible = "adi,i3c-master-v1" }, {} }; +MODULE_DEVICE_TABLE(of, adi_i3c_master_of_match); static int adi_i3c_master_probe(struct platform_device *pdev) { diff --git a/drivers/i3c/master/dw-i3c-master.c b/drivers/i3c/master/dw-i3c-master.c index 2f8c0c4683e06c..dc3b74822f8ea6 100644 --- a/drivers/i3c/master/dw-i3c-master.c +++ b/drivers/i3c/master/dw-i3c-master.c @@ -888,7 +888,15 @@ static int dw_i3c_master_daa(struct i3c_master_controller *m) if (!wait_for_completion_timeout(&xfer->comp, XFER_TIMEOUT)) dw_i3c_master_dequeue_xfer(master, xfer); - newdevs = GENMASK(master->maxdevs - cmd->rx_len - 1, 0); + /* + * cmd->rx_len holds the number of addresses ENTDAA left unassigned. + * On an empty bus rx_len == maxdevs, so avoid GENMASK(-1, 0). + */ + if (cmd->rx_len >= master->maxdevs) + newdevs = 0; + else + newdevs = GENMASK(master->maxdevs - cmd->rx_len - 1, 0); + newdevs &= ~olddevs; for (pos = 0; pos < master->maxdevs; pos++) { diff --git a/drivers/i3c/master/mipi-i3c-hci/pio.c b/drivers/i3c/master/mipi-i3c-hci/pio.c index ff2657ee220ba8..a1341d66bc656d 100644 --- a/drivers/i3c/master/mipi-i3c-hci/pio.c +++ b/drivers/i3c/master/mipi-i3c-hci/pio.c @@ -185,8 +185,11 @@ static void __hci_pio_init(struct i3c_hci *hci, u32 *size_val_ptr) pio_reg_write(INTR_SIGNAL_ENABLE, 0x0); pio_reg_write(INTR_STATUS_ENABLE, 0xffffffff); - /* Always accept error interrupts (will be activated on first xfer) */ - pio->enabled_irqs = STAT_ALL_ERRORS; + /* + * Always accept error interrupts and IBI threshold interrupt + * (will be activated on first xfer). + */ + pio->enabled_irqs = STAT_ALL_ERRORS | STAT_IBI_STATUS_THLD; } static void hci_pio_suspend(struct i3c_hci *hci) diff --git a/drivers/i3c/master/renesas-i3c.c b/drivers/i3c/master/renesas-i3c.c index c459e40fd5ffc3..6b403c3e58e9f5 100644 --- a/drivers/i3c/master/renesas-i3c.c +++ b/drivers/i3c/master/renesas-i3c.c @@ -663,6 +663,10 @@ static int renesas_i3c_daa(struct i3c_master_controller *m) if (!xfer) return -ENOMEM; + init_completion(&xfer->comp); + cmd = xfer->cmds; + cmd->rx_count = i3c->maxdevs; + /* Enable I3C bus. */ renesas_i3c_bus_enable(m, true); @@ -684,10 +688,6 @@ static int renesas_i3c_daa(struct i3c_master_controller *m) renesas_writel(i3c->regs, DATBAS(pos), datbas_dvdyad_with_parity(ret)); } - init_completion(&xfer->comp); - cmd = xfer->cmds; - cmd->rx_count = 0; - ret = renesas_i3c_get_free_pos(i3c); if (ret < 0) return ret; @@ -779,13 +779,13 @@ static int renesas_i3c_send_ccc_cmd(struct i3c_master_controller *m, if (!xfer) return -ENOMEM; - renesas_i3c_bus_enable(m, true); - init_completion(&xfer->comp); cmd = xfer->cmds; cmd->rnw = ccc->rnw; cmd->cmd0 = 0; + renesas_i3c_bus_enable(m, true); + /* Calculate the command descriptor. */ switch (ccc->id) { case I3C_CCC_SETDASA: @@ -835,15 +835,15 @@ static int renesas_i3c_i3c_xfers(struct i3c_dev_desc *dev, struct i3c_xfer *i3c_ struct renesas_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); int i; - /* Enable I3C bus. */ - renesas_i3c_bus_enable(m, true); - struct renesas_i3c_xfer *xfer __free(kfree) = renesas_i3c_alloc_xfer(i3c, 1); if (!xfer) return -ENOMEM; init_completion(&xfer->comp); + /* Enable I3C bus. */ + renesas_i3c_bus_enable(m, true); + for (i = 0; i < i3c_nxfers; i++) { struct renesas_i3c_cmd *cmd = xfer->cmds; @@ -957,19 +957,19 @@ static int renesas_i3c_i2c_xfers(struct i2c_dev_desc *dev, u8 start_bit = CNDCTL_STCND; int i; - struct renesas_i3c_xfer *xfer __free(kfree) = renesas_i3c_alloc_xfer(i3c, 1); - if (!xfer) - return -ENOMEM; - if (!i2c_nxfers) return 0; - renesas_i3c_bus_enable(m, false); + struct renesas_i3c_xfer *xfer __free(kfree) = renesas_i3c_alloc_xfer(i3c, 1); + if (!xfer) + return -ENOMEM; init_completion(&xfer->comp); xfer->is_i2c_xfer = true; cmd = xfer->cmds; + renesas_i3c_bus_enable(m, false); + if (!(renesas_readl(i3c->regs, BCST) & BCST_BFREF)) { cmd->err = -EBUSY; return cmd->err; diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index d74b478db28046..49c4ac9c7bb6eb 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -53,6 +53,7 @@ #include #include #include +#include #include #include #include @@ -2697,6 +2698,9 @@ error: pr_info("Failed to adjust C-states with data from 'intel_idle.table'\n"); } +#define INTEL_IDLE_INIT_QOS 20 +static struct pm_qos_request qos_req __initdata; + static int __init intel_idle_init(void) { const struct x86_cpu_id *id; @@ -2766,6 +2770,13 @@ static int __init intel_idle_init(void) if (retval) pr_warn("failed to initialized sysfs"); + /* + * Some platforms, in particular the Intel S1200BTL motherboard, have a + * problem with using package idle states too early, so prevent that + * from taking place until the device_initcall() phase is over. + */ + cpu_latency_qos_add_request(&qos_req, INTEL_IDLE_INIT_QOS); + retval = cpuidle_register_driver(&intel_idle_driver); if (retval) { struct cpuidle_driver *drv = cpuidle_get_driver(); @@ -2790,6 +2801,9 @@ hp_setup_fail: intel_idle_cpuidle_devices_uninit(); cpuidle_unregister_driver(&intel_idle_driver); init_driver_fail: + if (cpu_latency_qos_request_active((&qos_req))) + cpu_latency_qos_remove_request(&qos_req); + intel_idle_sysfs_uninit(); free_percpu(intel_idle_cpuidle_devices); return retval; @@ -2797,6 +2811,15 @@ init_driver_fail: } subsys_initcall_sync(intel_idle_init); +static int __init intel_idle_init_complete(void) +{ + if (cpu_latency_qos_request_active((&qos_req))) + cpu_latency_qos_remove_request(&qos_req); + + return 0; +} +device_initcall_sync(intel_idle_init_complete); + /* * We are not really modular, but we used to support that. Meaning we also * support "intel_idle.max_cstate=..." at boot and also a read-only export of diff --git a/drivers/iio/accel/dmard09.c b/drivers/iio/accel/dmard09.c index fe35a1270786bc..6f0497ab613353 100644 --- a/drivers/iio/accel/dmard09.c +++ b/drivers/iio/accel/dmard09.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #define DMARD09_DRV_NAME "dmard09" @@ -79,6 +80,12 @@ static int dmard09_read_raw(struct iio_dev *indio_dev, *val = accel; return IIO_VAL_INT; + case IIO_CHAN_INFO_SCALE: + *val = 0; + /* 1 g / 32 LSB, in m/s^2 */ + *val2 = IIO_G_TO_M_S_2(NANO / 32); + + return IIO_VAL_INT_PLUS_NANO; default: return -EINVAL; } diff --git a/drivers/iio/dac/ad5686-spi.c b/drivers/iio/dac/ad5686-spi.c index 8abfaf8f0c4692..859874ab861c5f 100644 --- a/drivers/iio/dac/ad5686-spi.c +++ b/drivers/iio/dac/ad5686-spi.c @@ -98,8 +98,13 @@ static const struct ad5686_bus_ops ad5686_spi_ops = { static int ad5686_spi_probe(struct spi_device *spi) { - return ad5686_probe(&spi->dev, spi_get_device_match_data(spi), - spi->modalias, &ad5686_spi_ops); + const struct ad5686_chip_info *info; + + info = spi_get_device_match_data(spi); + if (!info) + return -ENODATA; + + return ad5686_probe(&spi->dev, info, spi->modalias, &ad5686_spi_ops); } static const struct spi_device_id ad5686_spi_id[] = { diff --git a/drivers/iio/dac/ad5696-i2c.c b/drivers/iio/dac/ad5696-i2c.c index d49946adbde3a5..d5934405d5552b 100644 --- a/drivers/iio/dac/ad5696-i2c.c +++ b/drivers/iio/dac/ad5696-i2c.c @@ -68,8 +68,13 @@ static const struct ad5686_bus_ops ad5686_i2c_ops = { static int ad5686_i2c_probe(struct i2c_client *i2c) { - return ad5686_probe(&i2c->dev, i2c_get_match_data(i2c), - i2c->name, &ad5686_i2c_ops); + const struct ad5686_chip_info *info; + + info = i2c_get_match_data(i2c); + if (!info) + return -ENODATA; + + return ad5686_probe(&i2c->dev, info, i2c->name, &ad5686_i2c_ops); } static const struct i2c_device_id ad5686_i2c_id[] = { diff --git a/drivers/iio/light/gp2ap002.c b/drivers/iio/light/gp2ap002.c index 05773e24931beb..de29081fd3b20c 100644 --- a/drivers/iio/light/gp2ap002.c +++ b/drivers/iio/light/gp2ap002.c @@ -342,6 +342,10 @@ static int gp2ap002_write_event_config(struct iio_dev *indio_dev, bool state) { struct gp2ap002 *gp2ap002 = iio_priv(indio_dev); + int ret; + + if (state == gp2ap002->enabled) + return 0; if (state) { /* @@ -349,13 +353,16 @@ static int gp2ap002_write_event_config(struct iio_dev *indio_dev, * already) and reintialize the sensor by using runtime_pm * callbacks. */ - pm_runtime_get_sync(gp2ap002->dev); - gp2ap002->enabled = true; + ret = pm_runtime_resume_and_get(gp2ap002->dev); + if (ret) + return ret; + } else { pm_runtime_put_autosuspend(gp2ap002->dev); - gp2ap002->enabled = false; } + gp2ap002->enabled = state; + return 0; } @@ -642,6 +649,7 @@ static int gp2ap002_runtime_suspend(struct device *dev) /* Disable chip and IRQ, everything off */ ret = regmap_write(gp2ap002->map, GP2AP002_OPMOD, 0x00); if (ret) { + enable_irq(gp2ap002->irq); dev_err(gp2ap002->dev, "error setting up operation mode\n"); return ret; } diff --git a/drivers/iio/light/isl29028.c b/drivers/iio/light/isl29028.c index b88e7c4eae3ec5..6d5dd17e7a7d01 100644 --- a/drivers/iio/light/isl29028.c +++ b/drivers/iio/light/isl29028.c @@ -409,7 +409,7 @@ static int isl29028_write_raw(struct iio_dev *indio_dev, if (ret < 0) return ret; - return ret; + return 0; } static int isl29028_read_raw(struct iio_dev *indio_dev, diff --git a/drivers/iio/light/opt4060.c b/drivers/iio/light/opt4060.c index d248db1c42e8a3..f79dd342937d62 100644 --- a/drivers/iio/light/opt4060.c +++ b/drivers/iio/light/opt4060.c @@ -713,6 +713,7 @@ static ssize_t opt4060_read_ev_period(struct opt4060_chip *chip, int *val, { int ret, pers, fault_count, int_time; u64 uval; + u32 rem; int_time = opt4060_int_time_reg[chip->int_time][0]; @@ -738,7 +739,8 @@ static ssize_t opt4060_read_ev_period(struct opt4060_chip *chip, int *val, } uval = mul_u32_u32(int_time, pers); - *val = div_u64_rem(uval, MICRO, val2); + *val = div_u64_rem(uval, MICRO, &rem); + *val2 = rem; return IIO_VAL_INT_PLUS_MICRO; } diff --git a/drivers/iio/light/tsl2583.c b/drivers/iio/light/tsl2583.c index a0dd122af2cfb8..3b2c74d80e25c9 100644 --- a/drivers/iio/light/tsl2583.c +++ b/drivers/iio/light/tsl2583.c @@ -794,7 +794,7 @@ static int tsl2583_write_raw(struct iio_dev *indio_dev, if (ret < 0) return ret; - return ret; + return 0; } static const struct iio_info tsl2583_info = { diff --git a/drivers/iio/light/tsl2772.c b/drivers/iio/light/tsl2772.c index 244f44379c369d..2287585711c69a 100644 --- a/drivers/iio/light/tsl2772.c +++ b/drivers/iio/light/tsl2772.c @@ -1274,7 +1274,7 @@ static int tsl2772_read_raw(struct iio_dev *indio_dev, } break; case IIO_CHAN_INFO_CALIBSCALE: - if (chan->type == IIO_LIGHT) + if (chan->type == IIO_INTENSITY) *val = tsl2772_als_gain[chip->settings.als_gain]; else *val = tsl2772_prox_gain[chip->settings.prox_gain]; diff --git a/drivers/iio/orientation/hid-sensor-rotation.c b/drivers/iio/orientation/hid-sensor-rotation.c index cc3e66dbb90fcb..f761ec70b4c6cf 100644 --- a/drivers/iio/orientation/hid-sensor-rotation.c +++ b/drivers/iio/orientation/hid-sensor-rotation.c @@ -367,12 +367,6 @@ static int hid_dev_rot_probe(struct platform_device *pdev) return ret; } - ret = iio_device_register(indio_dev); - if (ret) { - dev_err(&pdev->dev, "device register failed\n"); - goto error_remove_trigger; - } - rot_state->callbacks.send_event = dev_rot_proc_event; rot_state->callbacks.capture_sample = dev_rot_capture_sample; rot_state->callbacks.pdev = pdev; @@ -380,13 +374,19 @@ static int hid_dev_rot_probe(struct platform_device *pdev) &rot_state->callbacks); if (ret) { dev_err(&pdev->dev, "callback reg failed\n"); - goto error_iio_unreg; + goto error_remove_trigger; + } + + ret = iio_device_register(indio_dev); + if (ret) { + dev_err(&pdev->dev, "device register failed\n"); + goto error_remove_callback; } return 0; -error_iio_unreg: - iio_device_unregister(indio_dev); +error_remove_callback: + sensor_hub_remove_callback(hsdev, hsdev->usage); error_remove_trigger: hid_sensor_remove_trigger(indio_dev, &rot_state->common_attributes); return ret; @@ -399,8 +399,8 @@ static void hid_dev_rot_remove(struct platform_device *pdev) struct iio_dev *indio_dev = platform_get_drvdata(pdev); struct dev_rot_state *rot_state = iio_priv(indio_dev); - sensor_hub_remove_callback(hsdev, hsdev->usage); iio_device_unregister(indio_dev); + sensor_hub_remove_callback(hsdev, hsdev->usage); hid_sensor_remove_trigger(indio_dev, &rot_state->common_attributes); } diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index e88d3efb967b3b..407310145e6c96 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -3528,10 +3528,12 @@ static void addr_handler(int status, struct sockaddr *src_addr, memcpy(addr, src_addr, rdma_addr_size(src_addr)); if (!status && !id_priv->cma_dev) { status = cma_acquire_dev_by_src_ip(id_priv); - if (status) + if (status) { pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to acquire device. status %d\n", status); - rdma_restrack_add(&id_priv->res); + } else { + rdma_restrack_add(&id_priv->res); + } } else if (status) { pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to resolve IP. status %d\n", status); } diff --git a/drivers/infiniband/core/counters.c b/drivers/infiniband/core/counters.c index a9e189194c1300..a2c85840c501a7 100644 --- a/drivers/infiniband/core/counters.c +++ b/drivers/infiniband/core/counters.c @@ -234,7 +234,6 @@ static void rdma_counter_free(struct rdma_counter *counter) mutex_unlock(&port_counter->lock); - rdma_restrack_del(&counter->res); rdma_free_hw_stats_struct(counter->stats); kfree(counter); } @@ -329,6 +328,7 @@ static void counter_release(struct kref *kref) counter = container_of(kref, struct rdma_counter, kref); counter_history_stat_update(counter); + rdma_restrack_del(&counter->res); counter->device->ops.counter_dealloc(counter); rdma_counter_free(counter); } @@ -490,7 +490,8 @@ static struct rdma_counter *rdma_get_counter_by_id(struct ib_device *dev, return NULL; counter = container_of(res, struct rdma_counter, res); - kref_get(&counter->kref); + if (!kref_get_unless_zero(&counter->kref)) + counter = NULL; rdma_restrack_put(res); return counter; diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c index 3d7b6cddd131c4..1379808e140409 100644 --- a/drivers/infiniband/core/cq.c +++ b/drivers/infiniband/core/cq.c @@ -327,6 +327,7 @@ void ib_free_cq(struct ib_cq *cq) if (WARN_ON_ONCE(cq->cqe_used)) return; + rdma_restrack_del(&cq->res); if (cq->device->ops.pre_destroy_cq) { ret = cq->device->ops.pre_destroy_cq(cq); WARN_ONCE(ret, "Disable of kernel CQ shouldn't fail"); @@ -353,7 +354,6 @@ void ib_free_cq(struct ib_cq *cq) else ret = cq->device->ops.destroy_cq(cq, NULL); WARN_ONCE(ret, "Destroy of kernel CQ shouldn't fail"); - rdma_restrack_del(&cq->res); kfree(cq->wc); kfree(cq); } diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index b8193e077a746f..d954eda6313492 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -3150,6 +3150,7 @@ static void __exit ib_core_cleanup(void) /* Make sure that any pending umem accounting work is done. */ destroy_workqueue(ib_wq); destroy_workqueue(ib_unreg_wq); + rcu_barrier(); WARN_ON(!xa_empty(&clients)); WARN_ON(!xa_empty(&devices)); } diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c index 02a0a9c0a4a6ad..f0f09670956d67 100644 --- a/drivers/infiniband/core/nldev.c +++ b/drivers/infiniband/core/nldev.c @@ -2133,6 +2133,11 @@ static int nldev_stat_set_counter_dynamic_doit(struct nlattr *tb[], nla_for_each_nested(entry_attr, tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS], rem) { + if (nla_len(entry_attr) != sizeof(u32)) { + ret = -EINVAL; + goto out; + } + index = nla_get_u32(entry_attr); if ((index >= stats->num_counters) || !(stats->descs[index].flags & IB_STAT_FLAG_OPTIONAL)) { diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c index cfee2071586c16..0d7e40f63c8a00 100644 --- a/drivers/infiniband/core/restrack.c +++ b/drivers/infiniband/core/restrack.c @@ -129,6 +129,46 @@ static void rdma_restrack_attach_task(struct rdma_restrack_entry *res, res->user = true; } +static struct rdma_restrack_root *res_to_rt(struct rdma_restrack_entry *res) +{ + struct ib_device *dev = res_to_dev(res); + + if (WARN_ON(!dev)) + return NULL; + + return &dev->res[res->type]; +} + +static void restrack_drain_res(struct rdma_restrack_root *rt, + struct rdma_restrack_entry *res) +{ + if (rt) { + struct rdma_restrack_entry *old; + + old = xa_cmpxchg(&rt->xa, res->id, res, XA_ZERO_ENTRY, + GFP_KERNEL); + WARN_ON(old != res); + } + + rdma_restrack_put(res); + wait_for_completion(&res->comp); +} + +static void restrack_restore_res(struct rdma_restrack_root *rt, + struct rdma_restrack_entry *res) +{ + reinit_completion(&res->comp); + kref_init(&res->kref); + + if (rt) { + struct rdma_restrack_entry *old; + + old = xa_cmpxchg(&rt->xa, res->id, XA_ZERO_ENTRY, res, + GFP_KERNEL); + WARN_ON(old); + } +} + /** * rdma_restrack_set_name() - set the task for this resource * @res: resource entry @@ -177,22 +217,23 @@ void rdma_restrack_new(struct rdma_restrack_entry *res, EXPORT_SYMBOL(rdma_restrack_new); /** - * rdma_restrack_add() - add object to the resource tracking database + * rdma_restrack_add() - add object to the resource tracking database. + * If this resource reuses an ID of a resource that was already destroyed + * after calling rdma_restrack_begin() but didn't yet call + * rdma_restrack_commit_del() it can result in an untracked QP. * @res: resource entry */ void rdma_restrack_add(struct rdma_restrack_entry *res) { - struct ib_device *dev = res_to_dev(res); struct rdma_restrack_root *rt; int ret = 0; - if (!dev) - return; - if (res->no_track) goto out; - rt = &dev->res[res->type]; + rt = res_to_rt(res); + if (!rt) + return; if (res->type == RDMA_RESTRACK_QP) { /* Special case to ensure that LQPN points to right QP */ @@ -229,6 +270,28 @@ out: } EXPORT_SYMBOL(rdma_restrack_add); +/** + * rdma_restrack_abort_del() - re-add object to the resource tracking database + * it can only be used after rdma_restrack_begin_del(). + * @res: resource entry + */ +void rdma_restrack_abort_del(struct rdma_restrack_entry *res) +{ + struct rdma_restrack_root *rt = NULL; + + if (!res->valid) + return; + + if (!res->no_track) { + rt = res_to_rt(res); + if (!rt) + return; + } + + restrack_restore_res(rt, res); +} +EXPORT_SYMBOL(rdma_restrack_abort_del); + int __must_check rdma_restrack_get(struct rdma_restrack_entry *res) { return kref_get_unless_zero(&res->kref); @@ -265,7 +328,7 @@ static void restrack_release(struct kref *kref) struct rdma_restrack_entry *res; res = container_of(kref, struct rdma_restrack_entry, kref); - if (res->task) { + if (res->task && !res->valid) { put_task_struct(res->task); res->task = NULL; } @@ -291,37 +354,20 @@ EXPORT_SYMBOL(rdma_restrack_put); */ void rdma_restrack_sync(struct rdma_restrack_entry *res) { - struct rdma_restrack_entry *old; struct rdma_restrack_root *rt; - struct task_struct *task; - struct ib_device *dev; if (!res->valid || res->no_track) return; - dev = res_to_dev(res); - if (WARN_ON(!dev)) + rt = res_to_rt(res); + if (!rt) return; - rt = &dev->res[res->type]; if (WARN_ON(xa_get_mark(&rt->xa, res->id, RESTRACK_DD))) return; - old = xa_cmpxchg(&rt->xa, res->id, res, XA_ZERO_ENTRY, GFP_KERNEL); - if (WARN_ON(old != res)) - return; - - task = res->task; - if (task) - get_task_struct(task); - rdma_restrack_put(res); - wait_for_completion(&res->comp); - reinit_completion(&res->comp); - if (task) - res->task = task; - kref_init(&res->kref); - - xa_cmpxchg(&rt->xa, res->id, XA_ZERO_ENTRY, res, GFP_KERNEL); + restrack_drain_res(rt, res); + restrack_restore_res(rt, res); } EXPORT_SYMBOL(rdma_restrack_sync); @@ -333,7 +379,6 @@ void rdma_restrack_del(struct rdma_restrack_entry *res) { struct rdma_restrack_entry *old; struct rdma_restrack_root *rt; - struct ib_device *dev; if (!res->valid) { if (res->task) { @@ -346,12 +391,10 @@ void rdma_restrack_del(struct rdma_restrack_entry *res) if (res->no_track) goto out; - dev = res_to_dev(res); - if (WARN_ON(!dev)) + rt = res_to_rt(res); + if (!rt) return; - rt = &dev->res[res->type]; - old = xa_erase(&rt->xa, res->id); WARN_ON(old != res); @@ -359,5 +402,61 @@ out: res->valid = false; rdma_restrack_put(res); wait_for_completion(&res->comp); + if (res->task) { + put_task_struct(res->task); + res->task = NULL; + } } EXPORT_SYMBOL(rdma_restrack_del); + +/** + * rdma_restrack_begin_del() - invalidate the object from the resource tracking + * database but preserve its index in the array. + * Since this preserves the index in the array until rdma_restrack_commit_del() + * is called, if rdma_restrack_add() is called in between with an old QP ID it + * can result in an untracked QP. + * @res: resource entry + */ +void rdma_restrack_begin_del(struct rdma_restrack_entry *res) +{ + struct rdma_restrack_root *rt = NULL; + + if (!res->valid) + return; + + if (!res->no_track) { + rt = res_to_rt(res); + if (!rt) + return; + } + + restrack_drain_res(rt, res); +} +EXPORT_SYMBOL(rdma_restrack_begin_del); + +/** + * rdma_restrack_commit_del() - delete object from the resource tracking + * database and free the task. + * @res: resource entry + */ +void rdma_restrack_commit_del(struct rdma_restrack_entry *res) +{ + struct rdma_restrack_root *rt; + + if (!res->valid || res->no_track) + goto out; + + rt = res_to_rt(res); + if (!rt) + return; + + xa_erase(&rt->xa, res->id); + +out: + res->valid = false; + if (res->task) { + put_task_struct(res->task); + res->task = NULL; + } +} +EXPORT_SYMBOL(rdma_restrack_commit_del); diff --git a/drivers/infiniband/core/restrack.h b/drivers/infiniband/core/restrack.h index 75b8d1005a984b..2df78e084e107c 100644 --- a/drivers/infiniband/core/restrack.h +++ b/drivers/infiniband/core/restrack.h @@ -26,8 +26,11 @@ struct rdma_restrack_root { int rdma_restrack_init(struct ib_device *dev); void rdma_restrack_clean(struct ib_device *dev); void rdma_restrack_add(struct rdma_restrack_entry *res); +void rdma_restrack_abort_del(struct rdma_restrack_entry *res); void rdma_restrack_del(struct rdma_restrack_entry *res); void rdma_restrack_sync(struct rdma_restrack_entry *res); +void rdma_restrack_begin_del(struct rdma_restrack_entry *res); +void rdma_restrack_commit_del(struct rdma_restrack_entry *res); void rdma_restrack_new(struct rdma_restrack_entry *res, enum rdma_restrack_type type); void rdma_restrack_set_name(struct rdma_restrack_entry *res, diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c index ac29dfa69bb3a6..4929636f7c5316 100644 --- a/drivers/infiniband/core/ucma.c +++ b/drivers/infiniband/core/ucma.c @@ -951,7 +951,7 @@ static ssize_t ucma_query_path(struct ucma_context *ctx, resp->num_paths = ctx->cm_id->route.num_pri_alt_paths; for (i = 0, out_len -= sizeof(*resp); - i < resp->num_paths && out_len > sizeof(struct ib_path_rec_data); + i < resp->num_paths && out_len >= sizeof(struct ib_path_rec_data); i++, out_len -= sizeof(struct ib_path_rec_data)) { struct sa_path_rec *rec = &ctx->cm_id->route.path_rec[i]; diff --git a/drivers/infiniband/core/uverbs_std_types_dmah.c b/drivers/infiniband/core/uverbs_std_types_dmah.c index 97101e0938263d..9873ab49a60132 100644 --- a/drivers/infiniband/core/uverbs_std_types_dmah.c +++ b/drivers/infiniband/core/uverbs_std_types_dmah.c @@ -18,11 +18,14 @@ static int uverbs_free_dmah(struct ib_uobject *uobject, if (atomic_read(&dmah->usecnt)) return -EBUSY; + rdma_restrack_begin_del(&dmah->res); ret = dmah->device->ops.dealloc_dmah(dmah, attrs); - if (ret) + if (ret) { + rdma_restrack_abort_del(&dmah->res); return ret; + } - rdma_restrack_del(&dmah->res); + rdma_restrack_commit_del(&dmah->res); kfree(dmah); return 0; } diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index 86811d31092ce0..f8b219bd308bd4 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -392,6 +392,7 @@ int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata) { int ret; + rdma_restrack_begin_del(&pd->res); if (pd->__internal_mr) { ret = pd->device->ops.dereg_mr(pd->__internal_mr, NULL); WARN_ON(ret); @@ -399,10 +400,12 @@ int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata) } ret = pd->device->ops.dealloc_pd(pd, udata); - if (ret) + if (ret) { + rdma_restrack_abort_del(&pd->res); return ret; + } - rdma_restrack_del(&pd->res); + rdma_restrack_commit_del(&pd->res); kfree(pd); return ret; } @@ -1140,16 +1143,20 @@ int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata) if (atomic_read(&srq->usecnt)) return -EBUSY; + rdma_restrack_begin_del(&srq->res); + ret = srq->device->ops.destroy_srq(srq, udata); - if (ret) + if (ret) { + rdma_restrack_abort_del(&srq->res); return ret; + } atomic_dec(&srq->pd->usecnt); if (srq->srq_type == IB_SRQT_XRC && srq->ext.xrc.xrcd) atomic_dec(&srq->ext.xrc.xrcd->usecnt); if (ib_srq_has_cq(srq->srq_type)) atomic_dec(&srq->ext.cq->usecnt); - rdma_restrack_del(&srq->res); + rdma_restrack_commit_del(&srq->res); kfree(srq); return ret; @@ -2154,6 +2161,8 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) if (qp->real_qp != qp) return __ib_destroy_shared_qp(qp); + rdma_restrack_begin_del(&qp->res); + sec = qp->qp_sec; if (sec) ib_destroy_qp_security_begin(sec); @@ -2166,6 +2175,7 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) if (ret) { if (sec) ib_destroy_qp_security_abort(sec); + rdma_restrack_abort_del(&qp->res); return ret; } @@ -2178,7 +2188,7 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) if (sec) ib_destroy_qp_security_end(sec); - rdma_restrack_del(&qp->res); + rdma_restrack_commit_del(&qp->res); kfree(qp); return ret; } @@ -2244,11 +2254,15 @@ int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata) if (atomic_read(&cq->usecnt)) return -EBUSY; + rdma_restrack_begin_del(&cq->res); + ret = cq->device->ops.destroy_cq(cq, udata); - if (ret) + if (ret) { + rdma_restrack_abort_del(&cq->res); return ret; + } - rdma_restrack_del(&cq->res); + rdma_restrack_commit_del(&cq->res); kfree(cq); return ret; } diff --git a/drivers/infiniband/hw/bng_re/bng_fw.c b/drivers/infiniband/hw/bng_re/bng_fw.c index 50156c300b335f..ab6a2d2e95b534 100644 --- a/drivers/infiniband/hw/bng_re/bng_fw.c +++ b/drivers/infiniband/hw/bng_re/bng_fw.c @@ -401,14 +401,15 @@ static int __wait_for_resp(struct bng_re_rcfw *rcfw, u16 cookie) { struct bng_re_cmdq_ctx *cmdq; struct bng_re_crsqe *crsqe; + unsigned long time_left; cmdq = &rcfw->cmdq; crsqe = &rcfw->crsqe_tbl[cookie]; do { - wait_event_timeout(cmdq->waitq, - !crsqe->is_in_used, - secs_to_jiffies(rcfw->max_timeout)); + time_left = wait_event_timeout(cmdq->waitq, + !crsqe->is_in_used, + secs_to_jiffies(rcfw->max_timeout)); if (!crsqe->is_in_used) return 0; @@ -417,6 +418,9 @@ static int __wait_for_resp(struct bng_re_rcfw *rcfw, u16 cookie) if (!crsqe->is_in_used) return 0; + + if (!time_left) + return -ENODEV; } while (true); }; diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 56576252900763..3890049e258224 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -695,7 +695,7 @@ int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) struct bnxt_re_dev *rdev = pd->rdev; int ret; - ret = ib_is_udata_in_empty(udata); + ret = ib_no_udata_io(udata); if (ret) return ret; @@ -712,7 +712,7 @@ int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) &pd->qplib_pd)) atomic_dec(&rdev->stats.res.pd_count); } - return ib_respond_empty_udata(udata); + return 0; } int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) @@ -844,7 +844,7 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr, u8 nw_type; int rc; - rc = ib_is_udata_in_empty(udata); + rc = ib_no_udata_io(udata); if (rc) return rc; @@ -901,7 +901,7 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr, if (active_ahs > rdev->stats.res.ah_watermark) rdev->stats.res.ah_watermark = active_ahs; - return ib_respond_empty_udata(udata); + return 0; } int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr) @@ -1015,7 +1015,7 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) unsigned int flags; int rc; - rc = ib_is_udata_in_empty(udata); + rc = ib_no_udata_io(udata); if (rc) return rc; @@ -1064,7 +1064,7 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) if (scq_nq != rcq_nq) bnxt_re_synchronize_nq(rcq_nq); - return ib_respond_empty_udata(udata); + return 0; } static u8 __from_ib_qp_type(enum ib_qp_type type) @@ -2148,7 +2148,7 @@ int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; int ret; - ret = ib_is_udata_in_empty(udata); + ret = ib_no_udata_io(udata); if (ret) return ret; @@ -2159,7 +2159,7 @@ int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) free_page((unsigned long)srq->uctx_srq_page); ib_umem_release(srq->umem); atomic_dec(&rdev->stats.res.srq_count); - return ib_respond_empty_udata(udata); + return 0; } static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev, @@ -2297,34 +2297,25 @@ int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr, { struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq); - struct bnxt_re_dev *rdev = srq->rdev; int ret; - ret = ib_is_udata_in_empty(udata); + ret = ib_no_udata_io(udata); if (ret) return ret; - switch (srq_attr_mask) { - case IB_SRQ_MAX_WR: - /* SRQ resize is not supported */ + if (srq_attr_mask != IB_SRQ_LIMIT) return -EINVAL; - case IB_SRQ_LIMIT: - /* Change the SRQ threshold */ - if (srq_attr->srq_limit > srq->qplib_srq.max_wqe) - return -EINVAL; - - srq->qplib_srq.threshold = srq_attr->srq_limit; - bnxt_qplib_srq_arm_db(&srq->qplib_srq.dbinfo, srq->qplib_srq.threshold); - /* On success, update the shadow */ - srq->srq_limit = srq_attr->srq_limit; - /* No need to Build and send response back to udata */ - return ib_respond_empty_udata(udata); - default: - ibdev_err(&rdev->ibdev, - "Unsupported srq_attr_mask 0x%x", srq_attr_mask); + if (srq_attr->srq_limit > srq->qplib_srq.max_wqe) return -EINVAL; - } + + srq->qplib_srq.threshold = srq_attr->srq_limit; + bnxt_qplib_srq_arm_db(&srq->qplib_srq.dbinfo, srq->qplib_srq.threshold); + + /* On success, update the shadow */ + srq->srq_limit = srq_attr->srq_limit; + /* No need to Build and send response back to udata */ + return 0; } int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr) @@ -2437,7 +2428,7 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, unsigned int flags; u8 nw_type; - rc = ib_is_udata_in_empty(udata); + rc = ib_no_udata_io(udata); if (rc) return rc; @@ -2689,7 +2680,7 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, if (rc) return rc; } - return ib_respond_empty_udata(udata); + return 0; } int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, @@ -3471,7 +3462,7 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) nq = cq->qplib_cq.nq; cctx = rdev->chip_ctx; - ret = ib_is_udata_in_empty(udata); + ret = ib_no_udata_io(udata); if (ret) return ret; @@ -3486,7 +3477,7 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) atomic_dec(&rdev->stats.res.cq_count); kfree(cq->cql); ib_umem_release(cq->umem); - return ib_respond_empty_udata(udata); + return 0; } int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, @@ -3688,6 +3679,10 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, unsigned int cqe, if (rc) goto fail; + rc = ib_respond_empty_udata(udata); + if (rc) + goto fail; + cq->resize_umem = ib_umem_get_va(&rdev->ibdev, req.cq_va, entries * sizeof(struct cq_base), IB_ACCESS_LOCAL_WRITE); @@ -3717,7 +3712,7 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, unsigned int cqe, cq->ib_cq.cqe = cq->resize_cqe; atomic_inc(&rdev->stats.res.resize_count); - return ib_respond_empty_udata(udata); + return 0; fail: if (cq->resize_umem) { @@ -4449,7 +4444,7 @@ int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) struct bnxt_re_dev *rdev = mr->rdev; int rc; - rc = ib_is_udata_in_empty(udata); + rc = ib_no_udata_io(udata); if (rc) return rc; @@ -4472,7 +4467,7 @@ int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) atomic_dec(&rdev->stats.res.mr_count); if (rc) return rc; - return ib_respond_empty_udata(udata); + return 0; } static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr) @@ -4991,11 +4986,13 @@ int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma) case BNXT_RE_MMAP_DBR_PAGE: case BNXT_RE_MMAP_TOGGLE_PAGE: /* Driver doesn't expect write access for user space */ - if (vma->vm_flags & VM_WRITE) + if (vma->vm_flags & VM_WRITE) { ret = -EFAULT; - else + } else { + vm_flags_clear(vma, VM_MAYWRITE); ret = vm_insert_page(vma, vma->vm_start, virt_to_page((void *)bnxt_entry->mem_offset)); + } break; default: ret = -EINVAL; diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c index c1815972aecff7..24ac9871ce67c5 100644 --- a/drivers/infiniband/hw/cxgb4/device.c +++ b/drivers/infiniband/hw/cxgb4/device.c @@ -933,6 +933,7 @@ static void c4iw_rdev_close(struct c4iw_rdev *rdev) void c4iw_dealloc(struct uld_ctx *ctx) { + debugfs_remove_recursive(ctx->dev->debugfs_root); c4iw_rdev_close(&ctx->dev->rdev); WARN_ON(!xa_empty(&ctx->dev->cqs)); WARN_ON(!xa_empty(&ctx->dev->qps)); @@ -957,7 +958,6 @@ static void c4iw_remove(struct uld_ctx *ctx) if (!ctx->dev) return; - debugfs_remove_recursive(ctx->dev->debugfs_root); c4iw_unregister_device(ctx->dev); c4iw_dealloc(ctx); } diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c index cd1b0101419842..ddb24a7fd4e6bd 100644 --- a/drivers/infiniband/hw/cxgb4/mem.c +++ b/drivers/infiniband/hw/cxgb4/mem.c @@ -199,7 +199,8 @@ static int _c4iw_write_mem_dma(struct c4iw_rdev *rdev, u32 addr, u32 len, daddr = dma_map_single(&rdev->lldi.pdev->dev, data, len, DMA_TO_DEVICE); if (dma_mapping_error(&rdev->lldi.pdev->dev, daddr)) - return -1; + return _c4iw_write_mem_inline(rdev, addr, len, data, skb, + wr_waitp); save = daddr; while (remain > inline_threshold) { @@ -235,30 +236,12 @@ static int write_adapter_mem(struct c4iw_rdev *rdev, u32 addr, u32 len, void *data, struct sk_buff *skb, struct c4iw_wr_wait *wr_waitp) { - int ret; - - if (!rdev->lldi.ulptx_memwrite_dsgl || !use_dsgl) { - ret = _c4iw_write_mem_inline(rdev, addr, len, data, skb, - wr_waitp); - goto out; - } - - if (len <= inline_threshold) { - ret = _c4iw_write_mem_inline(rdev, addr, len, data, skb, + if (!rdev->lldi.ulptx_memwrite_dsgl || !use_dsgl || + len <= inline_threshold) + return _c4iw_write_mem_inline(rdev, addr, len, data, skb, wr_waitp); - goto out; - } - - ret = _c4iw_write_mem_dma(rdev, addr, len, data, skb, wr_waitp); - if (ret) { - pr_warn_ratelimited("%s: dma map failure (non fatal)\n", - pci_name(rdev->lldi.pdev)); - ret = _c4iw_write_mem_inline(rdev, addr, len, data, skb, - wr_waitp); - } -out: - return ret; + return _c4iw_write_mem_dma(rdev, addr, len, data, skb, wr_waitp); } /* @@ -277,14 +260,19 @@ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry, int err; struct fw_ri_tpte *tpt; u32 stag_idx; + bool stag_idx_allocated = false; static atomic_t key; - if (c4iw_fatal_error(rdev)) + if (c4iw_fatal_error(rdev)) { + kfree_skb(skb); return -EIO; + } tpt = kmalloc_obj(*tpt); - if (!tpt) + if (!tpt) { + kfree_skb(skb); return -ENOMEM; + } stag_state = stag_state > 0; stag_idx = (*stag) >> 8; @@ -296,9 +284,11 @@ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry, rdev->stats.stag.fail++; mutex_unlock(&rdev->stats.lock); kfree(tpt); + kfree_skb(skb); return -ENOMEM; } mutex_lock(&rdev->stats.lock); + stag_idx_allocated = true; rdev->stats.stag.cur += 32; if (rdev->stats.stag.cur > rdev->stats.stag.max) rdev->stats.stag.max = rdev->stats.stag.cur; @@ -333,7 +323,7 @@ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry, (rdev->lldi.vr->stag.start >> 5), sizeof(*tpt), tpt, skb, wr_waitp); - if (reset_tpt_entry) { + if (reset_tpt_entry || (err && stag_idx_allocated)) { c4iw_put_resource(&rdev->resource.tpt_table, stag_idx); mutex_lock(&rdev->stats.lock); rdev->stats.stag.cur -= 32; @@ -469,8 +459,10 @@ struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc) FW_RI_STAG_NSMR, mhp->attr.perms, mhp->attr.mw_bind_enable, 0, 0, ~0ULL, 0, 0, 0, NULL, mhp->wr_waitp); - if (ret) - goto err_free_skb; + if (ret) { + kfree_skb(mhp->dereg_skb); + goto err_free_wr_wait; + } ret = finish_mem_reg(mhp, stag); if (ret) @@ -479,8 +471,6 @@ struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc) err_dereg_mem: dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, mhp->attr.pbl_addr, mhp->dereg_skb, mhp->wr_waitp); -err_free_skb: - kfree_skb(mhp->dereg_skb); err_free_wr_wait: c4iw_put_wr_wait(mhp->wr_waitp); err_free_mhp: diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 06d3365aeb5697..267a74ac15f3db 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -1345,9 +1345,11 @@ static int pbl_chunk_list_create(struct efa_dev *dev, struct pbl_context *pbl) chunk_list->chunks[i].length = EFA_CHUNK_USED_SIZE; } - chunk_list->chunks[chunk_list_size - 1].length = - ((page_cnt % EFA_PTRS_PER_CHUNK) * EFA_CHUNK_PAYLOAD_PTR_SIZE) + - EFA_CHUNK_PTR_SIZE; + + if (page_cnt % EFA_PTRS_PER_CHUNK != 0) + chunk_list->chunks[chunk_list_size - 1].length = + ((page_cnt % EFA_PTRS_PER_CHUNK) * EFA_CHUNK_PAYLOAD_PTR_SIZE) + + EFA_CHUNK_PTR_SIZE; /* fill the dma addresses of sg list pages to chunks: */ chunk_idx = 0; @@ -1359,9 +1361,12 @@ static int pbl_chunk_list_create(struct efa_dev *dev, struct pbl_context *pbl) rdma_block_iter_dma_address(&biter); if (payload_idx == EFA_PTRS_PER_CHUNK) { + payload_idx = 0; chunk_idx++; + if (chunk_idx >= chunk_list_size) + break; + cur_chunk_buf = chunk_list->chunks[chunk_idx].buf; - payload_idx = 0; } } diff --git a/drivers/infiniband/hw/erdma/Kconfig b/drivers/infiniband/hw/erdma/Kconfig index 267fc1f3c42af6..745e5551773f5b 100644 --- a/drivers/infiniband/hw/erdma/Kconfig +++ b/drivers/infiniband/hw/erdma/Kconfig @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only config INFINIBAND_ERDMA tristate "Alibaba Elastic RDMA Adapter (ERDMA) support" - depends on PCI_MSI && 64BIT + depends on PCI_MSI && 64BIT && !CPU_BIG_ENDIAN depends on INFINIBAND_ADDR_TRANS depends on INFINIBAND_USER_ACCESS help diff --git a/drivers/infiniband/hw/erdma/erdma_cm.c b/drivers/infiniband/hw/erdma/erdma_cm.c index 5c7d3a8f8038ce..70c5df566d9de6 100644 --- a/drivers/infiniband/hw/erdma/erdma_cm.c +++ b/drivers/infiniband/hw/erdma/erdma_cm.c @@ -1021,10 +1021,9 @@ int erdma_connect(struct iw_cm_id *id, struct iw_cm_conn_param *params) if (laddr->sa_family != AF_INET || raddr->sa_family != AF_INET) return -EAFNOSUPPORT; - qp = find_qp_by_qpn(dev, params->qpn); + qp = erdma_qp_get_by_qpn(dev, params->qpn); if (!qp) return -ENOENT; - erdma_qp_get(qp); ret = sock_create(AF_INET, SOCK_STREAM, IPPROTO_TCP, &s); if (ret < 0) @@ -1154,10 +1153,9 @@ int erdma_accept(struct iw_cm_id *id, struct iw_cm_conn_param *params) return -ECONNRESET; } - qp = find_qp_by_qpn(dev, params->qpn); + qp = erdma_qp_get_by_qpn(dev, params->qpn); if (!qp) return -ENOENT; - erdma_qp_get(qp); down_write(&qp->state_lock); if (qp->attrs.iwarp.state > ERDMA_QPS_IWARP_RTR) { diff --git a/drivers/infiniband/hw/erdma/erdma_eq.c b/drivers/infiniband/hw/erdma/erdma_eq.c index d5b9d19882b2a6..d5d1704cb57ca6 100644 --- a/drivers/infiniband/hw/erdma/erdma_eq.c +++ b/drivers/infiniband/hw/erdma/erdma_eq.c @@ -52,7 +52,7 @@ void erdma_aeq_event_handler(struct erdma_dev *dev) if (FIELD_GET(ERDMA_AEQE_HDR_TYPE_MASK, le32_to_cpu(aeqe->hdr)) == ERDMA_AE_TYPE_CQ_ERR) { cqn = le32_to_cpu(aeqe->event_data0); - cq = find_cq_by_cqn(dev, cqn); + cq = erdma_cq_get_by_cqn(dev, cqn); if (!cq) continue; @@ -62,9 +62,10 @@ void erdma_aeq_event_handler(struct erdma_dev *dev) if (cq->ibcq.event_handler) cq->ibcq.event_handler(&event, cq->ibcq.cq_context); + erdma_cq_put(cq); } else { qpn = le32_to_cpu(aeqe->event_data0); - qp = find_qp_by_qpn(dev, qpn); + qp = erdma_qp_get_by_qpn(dev, qpn); if (!qp) continue; @@ -74,6 +75,7 @@ void erdma_aeq_event_handler(struct erdma_dev *dev) if (qp->ibqp.event_handler) qp->ibqp.event_handler(&event, qp->ibqp.qp_context); + erdma_qp_put(qp); } } @@ -157,7 +159,7 @@ void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb) poll_cnt++; cqn = FIELD_GET(ERDMA_CEQE_HDR_CQN_MASK, READ_ONCE(*ceqe)); - cq = find_cq_by_cqn(dev, cqn); + cq = erdma_cq_get_by_cqn(dev, cqn); if (!cq) continue; @@ -166,6 +168,7 @@ void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb) if (cq->ibcq.comp_handler) cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); + erdma_cq_put(cq); } notify_eq(&ceq_cb->eq); @@ -220,6 +223,7 @@ static void erdma_free_ceq_irq(struct erdma_dev *dev, u16 ceqn) irq_set_affinity_hint(eqc->irq.msix_vector, NULL); free_irq(eqc->irq.msix_vector, eqc); + tasklet_kill(&eqc->tasklet); } static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq) diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index 74afe6eb18b0bc..20944d506da772 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -1304,8 +1304,15 @@ int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, true); + /* + * A timeout disables the command queue, so retry cannot succeed. Treat + * terminal command failures as diagnostic; propagating them can make + * forced uverbs cleanup discard the last software resource pointers. + */ if (ret) - return ret; + ibdev_warn_ratelimited(&dev->ibdev, + "failed to deregister MR 0x%x: %d\n", + ibmr->lkey, ret); erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8); @@ -1321,6 +1328,7 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) struct erdma_dev *dev = to_edev(ibcq->device); struct erdma_ucontext *ctx = rdma_udata_to_drv_context( udata, struct erdma_ucontext, ibucontext); + unsigned long flags; int err; struct erdma_cmdq_destroy_cq_req req; @@ -1331,7 +1339,16 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, true); if (err) - return err; + ibdev_warn_ratelimited(&dev->ibdev, + "failed to destroy CQ %u: %d\n", + cq->cqn, err); + + xa_lock_irqsave(&dev->cq_xa, flags); + __xa_erase(&dev->cq_xa, cq->cqn); + xa_unlock_irqrestore(&dev->cq_xa, flags); + + erdma_cq_put(cq); + wait_for_completion(&cq->free); if (rdma_is_kernel_res(&cq->ibcq.res)) { dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, @@ -1343,8 +1360,6 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) put_mtt_entries(dev, &cq->user_cq.qbuf_mem); } - xa_erase(&dev->cq_xa, cq->cqn); - return 0; } @@ -1356,6 +1371,7 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) udata, struct erdma_ucontext, ibucontext); struct erdma_cmdq_destroy_qp_req req; union erdma_mod_qp_params params; + unsigned long flags; int err; down_write(&qp->state_lock); @@ -1379,7 +1395,13 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, true); if (err) - return err; + ibdev_warn_ratelimited(&dev->ibdev, + "failed to destroy QP %u: %d\n", + QP_ID(qp), err); + + xa_lock_irqsave(&dev->qp_xa, flags); + __xa_erase(&dev->qp_xa, QP_ID(qp)); + xa_unlock_irqrestore(&dev->qp_xa, flags); erdma_qp_put(qp); wait_for_completion(&qp->safe_free); @@ -1394,7 +1416,6 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) if (qp->cep) erdma_cep_put(qp->cep); - xa_erase(&dev->qp_xa, QP_ID(qp)); return 0; } @@ -1971,6 +1992,8 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, cq->ibcq.cqe = depth; cq->depth = depth; cq->assoc_eqn = attr->comp_vector + 1; + refcount_set(&cq->refcount, 1); + init_completion(&cq->free); ret = xa_alloc_cyclic(&dev->cq_xa, &cq->cqn, cq, XA_LIMIT(1, dev->attrs.max_cq - 1), @@ -2281,7 +2304,9 @@ int erdma_destroy_ah(struct ib_ah *ibah, u32 flags) ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, flags & RDMA_DESTROY_AH_SLEEPABLE); if (ret) - return ret; + ibdev_warn_ratelimited(&dev->ibdev, + "failed to destroy AH %u: %d\n", + ah->ahn, ret); erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_AH], ah->ahn); diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h index 7d8d3fe501d5cd..c73cecf92f611a 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.h +++ b/drivers/infiniband/hw/erdma/erdma_verbs.h @@ -7,6 +7,9 @@ #ifndef __ERDMA_VERBS_H__ #define __ERDMA_VERBS_H__ +#include +#include + #include "erdma.h" /* RDMA Capability. */ @@ -341,6 +344,8 @@ struct erdma_cq { u32 depth; u32 assoc_eqn; + refcount_t refcount; + struct completion free; union { struct erdma_kcq_info kern_cq; @@ -355,9 +360,40 @@ static inline struct erdma_qp *find_qp_by_qpn(struct erdma_dev *dev, int id) return (struct erdma_qp *)xa_load(&dev->qp_xa, id); } -static inline struct erdma_cq *find_cq_by_cqn(struct erdma_dev *dev, int id) +static inline struct erdma_qp *erdma_qp_get_by_qpn(struct erdma_dev *dev, + int id) +{ + struct erdma_qp *qp; + unsigned long flags; + + xa_lock_irqsave(&dev->qp_xa, flags); + qp = xa_load(&dev->qp_xa, id); + if (qp && !kref_get_unless_zero(&qp->ref)) + qp = NULL; + xa_unlock_irqrestore(&dev->qp_xa, flags); + + return qp; +} + +static inline struct erdma_cq *erdma_cq_get_by_cqn(struct erdma_dev *dev, + int id) +{ + struct erdma_cq *cq; + unsigned long flags; + + xa_lock_irqsave(&dev->cq_xa, flags); + cq = xa_load(&dev->cq_xa, id); + if (cq && !refcount_inc_not_zero(&cq->refcount)) + cq = NULL; + xa_unlock_irqrestore(&dev->cq_xa, flags); + + return cq; +} + +static inline void erdma_cq_put(struct erdma_cq *cq) { - return (struct erdma_cq *)xa_load(&dev->cq_xa, id); + if (refcount_dec_and_test(&cq->refcount)) + complete(&cq->free); } void erdma_qp_get(struct erdma_qp *qp); diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c index b7fd8b1fbbbde7..c5f005e5b812cf 100644 --- a/drivers/infiniband/hw/hfi1/init.c +++ b/drivers/infiniband/hw/hfi1/init.c @@ -1225,8 +1225,9 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev, GFP_KERNEL); if (ret < 0) { dev_err(&pdev->dev, - "Could not allocate unit ID: error %d\n", -ret); - goto bail; + "Could not allocate unit ID: error %pe\n", ERR_PTR(ret)); + rvt_dealloc_device(&dd->verbs_dev.rdi); + return ERR_PTR(ret); } /* @@ -1655,11 +1656,8 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) * we still create devices, so diags, etc. can be used * to determine cause of problem. */ - if (!initfail && !ret) { + if (!initfail && !ret) dd->flags |= HFI1_INITTED; - /* create debufs files after init and ib register */ - hfi1_dbg_ibdev_init(&dd->verbs_dev); - } j = hfi1_device_create(dd); if (j) @@ -1668,7 +1666,6 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) if (initfail || ret) { msix_clean_up_interrupts(dd); stop_timers(dd); - flush_workqueue(ib_wq); for (pidx = 0; pidx < dd->num_pports; ++pidx) { hfi1_quiet_serdes(dd->pport + pidx); ppd = dd->pport + pidx; @@ -1685,6 +1682,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) hfi1_device_remove(dd); if (!ret) hfi1_unregister_ib_device(dd); + hfi1_free_rx(dd); postinit_cleanup(dd); if (initfail) ret = initfail; @@ -1692,6 +1690,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) } sdma_start(dd); + hfi1_dbg_ibdev_init(&dd->verbs_dev); return 0; @@ -1741,9 +1740,6 @@ static void remove_one(struct pci_dev *pdev) stop_timers(dd); - /* wait until all of our (qsfp) queue_work() calls complete */ - flush_workqueue(ib_wq); - postinit_cleanup(dd); } diff --git a/drivers/infiniband/hw/hfi1/user_sdma.c b/drivers/infiniband/hw/hfi1/user_sdma.c index 8ea5ed918a0289..be6b82ba93afc3 100644 --- a/drivers/infiniband/hw/hfi1/user_sdma.c +++ b/drivers/infiniband/hw/hfi1/user_sdma.c @@ -1026,6 +1026,7 @@ static int set_txreq_header_ahg(struct user_sdma_request *req, struct user_sdma_txreq *tx, u32 datalen) { u32 ahg[AHG_KDETH_ARRAY_SIZE]; + int ret; int idx = 0; u8 omfactor; /* KDETH.OM */ struct hfi1_user_sdma_pkt_q *pq = req->pq; @@ -1130,11 +1131,13 @@ static int set_txreq_header_ahg(struct user_sdma_request *req, trace_hfi1_sdma_user_header_ahg(pq->dd, pq->ctxt, pq->subctxt, req->info.comp_idx, req->sde->this_idx, req->ahg_idx, ahg, idx, tidval); - sdma_txinit_ahg(&tx->txreq, - SDMA_TXREQ_F_USE_AHG, - datalen, req->ahg_idx, idx, - ahg, sizeof(req->hdr), - user_sdma_txreq_cb); + ret = sdma_txinit_ahg(&tx->txreq, + SDMA_TXREQ_F_USE_AHG, + datalen, req->ahg_idx, idx, + ahg, sizeof(req->hdr), + user_sdma_txreq_cb); + if (ret) + return ret; return idx; } diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c index e4037d5ef8993e..290ad02ed65754 100644 --- a/drivers/infiniband/hw/irdma/utils.c +++ b/drivers/infiniband/hw/irdma/utils.c @@ -1168,6 +1168,12 @@ void irdma_free_qp_rsrc(struct irdma_qp *iwqp) iwqp->kqp.dma_mem.va = NULL; kfree(iwqp->kqp.sq_wrid_mem); kfree(iwqp->kqp.rq_wrid_mem); + + if (iwqp->user_mode && iwqp->iwpbl) { + struct irdma_mr *iwmr = iwqp->iwpbl->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } } /** diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index b7388b41ed9589..04d5af78686b5f 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -464,6 +464,9 @@ static struct irdma_pbl *irdma_get_pbl(unsigned long va, list_for_each_entry (iwpbl, pbl_list, list) { if (iwpbl->user_base == va) { + struct irdma_mr *iwmr = iwpbl->iwmr; + + refcount_inc(&iwmr->user_ring_refs); list_del(&iwpbl->list); iwpbl->on_list = false; return iwpbl; @@ -1880,6 +1883,11 @@ static void irdma_srq_free_rsrc(struct irdma_pci_f *rf, struct irdma_srq *iwsrq) dma_free_coherent(rf->sc_dev.hw->device, iwsrq->kmem.size, iwsrq->kmem.va, iwsrq->kmem.pa); iwsrq->kmem.va = NULL; + } else { + /* Not called in any failure path, so iwpbl is valid. */ + struct irdma_mr *iwmr = iwsrq->iwpbl->iwmr; + + refcount_dec(&iwmr->user_ring_refs); } irdma_free_rsrc(rf, rf->allocated_srqs, srq->srq_uk.srq_id); @@ -1902,6 +1910,21 @@ static void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq) iwcq->kmem_shadow.size, iwcq->kmem_shadow.va, iwcq->kmem_shadow.pa); iwcq->kmem_shadow.va = NULL; + } else { + struct irdma_mr *iwmr; + + /* May be called in a failure path before iwpbl is valid. */ + if (iwcq->iwpbl) { + iwmr = iwcq->iwpbl->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } + + if (iwcq->iwpbl_shadow) { + iwmr = iwcq->iwpbl_shadow->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } } irdma_free_rsrc(rf, rf->allocated_cqs, cq->cq_uk.cq_id); @@ -2017,7 +2040,7 @@ static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries, struct irdma_modify_cq_info info = {}; struct irdma_dma_mem kmem_buf; struct irdma_cq_mr *cqmr_buf; - struct irdma_pbl *iwpbl_buf; + struct irdma_pbl *iwpbl_buf = NULL; struct irdma_device *iwdev; struct irdma_pci_f *rf; struct irdma_cq_buf *cq_buf = NULL; @@ -2128,6 +2151,19 @@ static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries, goto error; spin_lock_irqsave(&iwcq->lock, flags); + if (udata) { + struct irdma_pbl *old_iwpbl = iwcq->iwpbl; + + /* Only update if the resize was successful. Otherwise, HW is + * still pointing to the old PBL. + */ + iwcq->iwpbl = iwpbl_buf; + if (old_iwpbl) { + struct irdma_mr *old_iwmr = old_iwpbl->iwmr; + + refcount_dec(&old_iwmr->user_ring_refs); + } + } if (cq_buf) { cq_buf->kmem_buf = iwcq->kmem; cq_buf->hw = dev->hw; @@ -2143,6 +2179,11 @@ static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries, return 0; error: + if (iwpbl_buf) { + struct irdma_mr *iwmr = iwpbl_buf->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } if (!udata) { dma_free_coherent(dev->hw->device, kmem_buf.size, kmem_buf.va, kmem_buf.pa); @@ -2419,6 +2460,11 @@ free_dmem: dma_free_coherent(rf->hw.device, iwsrq->kmem.size, iwsrq->kmem.va, iwsrq->kmem.pa); free_rsrc: + if (iwsrq->user_mode && iwsrq->iwpbl) { + struct irdma_mr *iwmr = iwsrq->iwpbl->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } irdma_free_rsrc(rf, rf->allocated_srqs, iwsrq->srq_num); return err_code; } @@ -2498,6 +2544,8 @@ static int irdma_create_cq(struct ib_cq *ibcq, INIT_LIST_HEAD(&iwcq->resize_list); INIT_LIST_HEAD(&iwcq->cmpl_generated); iwcq->cq_num = cq_num; + iwcq->iwpbl = NULL; + iwcq->iwpbl_shadow = NULL; info.dev = dev; ukinfo->cq_size = max(entries, 4); ukinfo->cq_id = cq_num; @@ -2517,8 +2565,6 @@ static int irdma_create_cq(struct ib_cq *ibcq, struct irdma_ucontext *ucontext; struct irdma_create_cq_req req = {}; struct irdma_cq_mr *cqmr; - struct irdma_pbl *iwpbl; - struct irdma_pbl *iwpbl_shadow; struct irdma_cq_mr *cqmr_shadow; iwcq->user_mode = true; @@ -2532,34 +2578,34 @@ static int irdma_create_cq(struct ib_cq *ibcq, } spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); - iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, - &ucontext->cq_reg_mem_list); + iwcq->iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, + &ucontext->cq_reg_mem_list); spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); - if (!iwpbl) { + if (!iwcq->iwpbl) { err_code = -EPROTO; goto cq_free_rsrc; } - cqmr = &iwpbl->cq_mr; + cqmr = &iwcq->iwpbl->cq_mr; if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE) { spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); - iwpbl_shadow = irdma_get_pbl( + iwcq->iwpbl_shadow = irdma_get_pbl( (unsigned long)req.user_shadow_area, &ucontext->cq_reg_mem_list); spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); - if (!iwpbl_shadow) { + if (!iwcq->iwpbl_shadow) { err_code = -EPROTO; goto cq_free_rsrc; } - cqmr_shadow = &iwpbl_shadow->cq_mr; + cqmr_shadow = &iwcq->iwpbl_shadow->cq_mr; info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; } else { info.shadow_area_pa = cqmr->shadow; } - if (iwpbl->pbl_allocated) { + if (iwcq->iwpbl->pbl_allocated) { info.virtual_map = true; info.pbl_chunk_size = 1; info.first_pm_pbl_idx = cqmr->cq_pbl.idx; @@ -3362,6 +3408,7 @@ static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region, if (!iwmr) return ERR_PTR(-ENOMEM); + refcount_set(&iwmr->user_ring_refs, 1); iwpbl = &iwmr->iwpbl; iwpbl->iwmr = iwmr; iwmr->region = region; @@ -3929,41 +3976,41 @@ static struct ib_mr *irdma_get_dma_mr(struct ib_pd *pd, int acc) * irdma_del_memlist - Deleting pbl list entries for CQ/QP * @iwmr: iwmr for IB's user page addresses * @ucontext: ptr to user context + * + * Return: True if the MR is currently in-use by a QP/CQ/SRQ ring. */ -static void irdma_del_memlist(struct irdma_mr *iwmr, +static bool irdma_del_memlist(struct irdma_mr *iwmr, struct irdma_ucontext *ucontext) { struct irdma_pbl *iwpbl = &iwmr->iwpbl; unsigned long flags; + spinlock_t *lock; + bool in_use = false; switch (iwmr->type) { case IRDMA_MEMREG_TYPE_CQ: - spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); - if (iwpbl->on_list) { - iwpbl->on_list = false; - list_del(&iwpbl->list); - } - spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); + lock = &ucontext->cq_reg_mem_list_lock; break; case IRDMA_MEMREG_TYPE_QP: - spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); - if (iwpbl->on_list) { - iwpbl->on_list = false; - list_del(&iwpbl->list); - } - spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); + lock = &ucontext->qp_reg_mem_list_lock; break; case IRDMA_MEMREG_TYPE_SRQ: - spin_lock_irqsave(&ucontext->srq_reg_mem_list_lock, flags); - if (iwpbl->on_list) { - iwpbl->on_list = false; - list_del(&iwpbl->list); - } - spin_unlock_irqrestore(&ucontext->srq_reg_mem_list_lock, flags); + lock = &ucontext->srq_reg_mem_list_lock; break; default: - break; + return false; } + + spin_lock_irqsave(lock, flags); + if (!refcount_dec_if_one(&iwmr->user_ring_refs)) { + in_use = true; + } else if (iwpbl->on_list) { + iwpbl->on_list = false; + list_del(&iwpbl->list); + } + spin_unlock_irqrestore(lock, flags); + + return in_use; } /** @@ -3986,7 +4033,12 @@ static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); - irdma_del_memlist(iwmr, ucontext); + + /* Do not allow the MR to be unpinned if it is still + * backing a user ring. + */ + if (irdma_del_memlist(iwmr, ucontext)) + return -EBUSY; } goto done; } diff --git a/drivers/infiniband/hw/irdma/verbs.h b/drivers/infiniband/hw/irdma/verbs.h index 289ebc9b23ca78..a1651641eb7146 100644 --- a/drivers/infiniband/hw/irdma/verbs.h +++ b/drivers/infiniband/hw/irdma/verbs.h @@ -120,6 +120,7 @@ struct irdma_mr { u64 len; u64 pgaddrmem[IRDMA_MAX_SAVED_PHY_PGADDR]; struct irdma_pbl iwpbl; + refcount_t user_ring_refs; }; struct irdma_srq { @@ -152,6 +153,8 @@ struct irdma_cq { struct list_head resize_list; struct irdma_cq_poll_info cur_cqe; struct list_head cmpl_generated; + struct irdma_pbl *iwpbl; + struct irdma_pbl *iwpbl_shadow; }; struct irdma_cmpl_gen { diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index 60926f39ab9da3..389dad4ae1ff09 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -472,6 +472,12 @@ static void mana_table_remove_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp xa_erase_irq(&mdev->qp_table_wq, qp->ibqp.qp_num); } +static void mana_table_drain_qp_ref(struct mana_ib_qp *qp) +{ + mana_put_qp_ref(qp); + wait_for_completion(&qp->free); +} + static int mana_table_store_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) { u32 qids = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].id | MANA_SENDQ_MASK; @@ -490,6 +496,7 @@ static int mana_table_store_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *q remove_sq: xa_erase_irq(&mdev->qp_table_wq, qids); + mana_table_drain_qp_ref(qp); return err; } @@ -537,8 +544,7 @@ static void mana_table_remove_qp(struct mana_ib_dev *mdev, qp->ibqp.qp_type); return; } - mana_put_qp_ref(qp); - wait_for_completion(&qp->free); + mana_table_drain_qp_ref(qp); } static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, diff --git a/drivers/infiniband/hw/mlx5/cong.c b/drivers/infiniband/hw/mlx5/cong.c index d0edf83a2f20f8..42d005a5a7df44 100644 --- a/drivers/infiniband/hw/mlx5/cong.c +++ b/drivers/infiniband/hw/mlx5/cong.c @@ -361,7 +361,7 @@ static int mlx5_ib_set_cc_params(struct mlx5_ib_dev *dev, u32 port_num, MLX5_SET(field_select_r_roce_rp, field, field_select_r_roce_rp, attr_mask); - err = mlx5_cmd_exec_in(dev->mdev, modify_cong_params, in); + err = mlx5_cmd_exec_in(mdev, modify_cong_params, in); kvfree(in); alloc_err: mlx5_ib_put_native_port_mdev(dev, port_num + 1); @@ -399,15 +399,13 @@ static ssize_t get_param(struct file *filp, char __user *buf, size_t count, int offset = param->offset; u32 var = 0; int ret; - char lbuf[11]; + char lbuf[12]; ret = mlx5_ib_get_cc_params(param->dev, param->port_num, offset, &var); if (ret) return ret; - ret = snprintf(lbuf, sizeof(lbuf), "%d\n", var); - if (ret < 0) - return ret; + ret = scnprintf(lbuf, sizeof(lbuf), "%u\n", var); return simple_read_from_buffer(buf, count, pos, lbuf, ret); } diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 02809114fc79af..c283a902ea7e47 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -1631,14 +1631,15 @@ static int mlx5_ib_query_port_speed_from_vport(struct mlx5_core_dev *mdev, u32 port_num) { u32 max_tx_speed; + u8 vport_state; int err; err = mlx5_query_vport_max_tx_speed(mdev, op_mod, vport, other_vport, - &max_tx_speed); + &max_tx_speed, &vport_state); if (err) return err; - if (max_tx_speed == 0) + if (vport_state == VPORT_STATE_DOWN || max_tx_speed == 0) /* Value 0 indicates field not supported, fallback */ return mlx5_ib_query_port_speed_from_port(dev, port_num, speed); @@ -5538,6 +5539,7 @@ mp_err: dd_err: mlx5r_rep_cleanup(); rep_err: + rcu_barrier(); mlx5_ib_qp_event_cleanup(); qp_event_err: destroy_workqueue(mlx5_ib_event_wq); @@ -5551,6 +5553,7 @@ static void __exit mlx5_ib_cleanup(void) auxiliary_driver_unregister(&mlx5r_driver); auxiliary_driver_unregister(&mlx5r_mp_driver); mlx5r_rep_cleanup(); + rcu_barrier(); mlx5_ib_qp_event_cleanup(); destroy_workqueue(mlx5_ib_event_wq); diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 7ff02d89c31d5b..e25ac139e43f09 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -647,6 +647,7 @@ static int set_user_buf_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr) { int desc_sz = 1 << qp->sq.wqe_shift; + int rq_buf_size, sq_buf_size; if (desc_sz > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) { mlx5_ib_warn(dev, "desc_sz %d, max_sq_desc_sz %d\n", @@ -671,11 +672,21 @@ static int set_user_buf_size(struct mlx5_ib_dev *dev, if (attr->qp_type == IB_QPT_RAW_PACKET || qp->flags & IB_QP_CREATE_SOURCE_QPN) { - base->ubuffer.buf_size = qp->rq.wqe_cnt << qp->rq.wqe_shift; - qp->raw_packet_qp.sq.ubuffer.buf_size = qp->sq.wqe_cnt << 6; + if (check_shl_overflow(qp->rq.wqe_cnt, qp->rq.wqe_shift, + &base->ubuffer.buf_size)) + return -EINVAL; + if (check_shl_overflow(qp->sq.wqe_cnt, 6, + &qp->raw_packet_qp.sq.ubuffer.buf_size)) + return -EINVAL; } else { - base->ubuffer.buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + - (qp->sq.wqe_cnt << 6); + if (check_shl_overflow(qp->rq.wqe_cnt, qp->rq.wqe_shift, + &rq_buf_size)) + return -EINVAL; + if (check_shl_overflow(qp->sq.wqe_cnt, 6, &sq_buf_size)) + return -EINVAL; + if (check_add_overflow(rq_buf_size, sq_buf_size, + &base->ubuffer.buf_size)) + return -EINVAL; } return 0; @@ -1004,7 +1015,11 @@ static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, qp->rq.offset = 0; qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB); - qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift; + if (check_shl_overflow(qp->rq.wqe_cnt, qp->rq.wqe_shift, + &qp->sq.offset)) { + err = -EINVAL; + goto err_bfreg; + } err = set_user_buf_size(dev, qp, ucmd, base, attr); if (err) diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c index 5fa3a1f3332689..f37d6d64adb9d2 100644 --- a/drivers/infiniband/sw/rdmavt/vt.c +++ b/drivers/infiniband/sw/rdmavt/vt.c @@ -55,8 +55,10 @@ struct rvt_dev_info *rvt_alloc_device(size_t size, int nports) return rdi; rdi->ports = kzalloc_objs(*rdi->ports, nports); - if (!rdi->ports) + if (!rdi->ports) { ib_dealloc_device(&rdi->ibdev); + return NULL; + } return rdi; } diff --git a/drivers/infiniband/sw/rxe/rxe_odp.c b/drivers/infiniband/sw/rxe/rxe_odp.c index ff904d5e54a734..1b1c4a0c711005 100644 --- a/drivers/infiniband/sw/rxe/rxe_odp.c +++ b/drivers/infiniband/sw/rxe/rxe_odp.c @@ -114,6 +114,7 @@ int rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, err = rxe_odp_init_pages(mr); if (err) { ib_umem_odp_release(umem_odp); + mr->umem = NULL; return err; } diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index 12d03f390b0978..24f5c044363f77 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -701,6 +701,21 @@ int rxe_requester(struct rxe_qp *qp) if (unlikely(!wqe)) goto exit; + /* + * Don't trust user space data: a user QP's WQE comes from an mmap'd + * ring, so num_sge/cur_sge are attacker-controlled. Bound num_sge like + * get_srq_wqe(); bound cur_sge only when payload exists (dma.resid), + * since copy_data() skips dma->sge[] on a zero-length copy (all a + * max_sge == 0 QP can post). + */ + if (unlikely(wqe->dma.num_sge > qp->sq.max_sge || + (wqe->dma.resid && + wqe->dma.cur_sge >= qp->sq.max_sge))) { + rxe_dbg_qp(qp, "invalid num_sge/cur_sge in send wqe\n"); + wqe->status = IB_WC_LOC_QP_OP_ERR; + goto err; + } + if (rxe_wqe_is_fenced(qp, wqe)) { qp->req.wait_fence = 1; goto exit; diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index d8cbdfa70cdbdd..02b16e2b49b8f9 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -1217,7 +1217,14 @@ finish: spin_lock_irqsave(&qp->state_lock, flags); if (unlikely(qp_state(qp) == IB_QPS_ERR)) { spin_unlock_irqrestore(&qp->state_lock, flags); - return RESPST_CHK_RESOURCE; + /* The packet was executed and completed before the QP + * moved to ERROR; it must be consumed exactly once. + * Re-entering the request chain with the stale packet + * would copy it into every remaining recv WQE as a new + * completion. Remaining WQEs are flushed by the drain + * path at rxe_receiver() entry. + */ + return pkt ? RESPST_CLEANUP : RESPST_CHK_RESOURCE; } spin_unlock_irqrestore(&qp->state_lock, flags); diff --git a/drivers/infiniband/sw/siw/siw_cm.c b/drivers/infiniband/sw/siw/siw_cm.c index 87c79527ac0903..0245b25e727189 100644 --- a/drivers/infiniband/sw/siw/siw_cm.c +++ b/drivers/infiniband/sw/siw/siw_cm.c @@ -1751,7 +1751,7 @@ error: cep->state = SIW_EPSTATE_CLOSED; siw_free_cm_id(cep); - if (qp->cep) { + if (qp->cep == cep) { siw_cep_put(cep); qp->cep = NULL; } diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c index 16a015b6720632..6c14246befb1f9 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -2783,6 +2783,7 @@ err_client: err_sa: ib_sa_unregister_client(&ipoib_sa_client); destroy_workqueue(ipoib_workqueue); + rcu_barrier(); err_fs: ipoib_unregister_debugfs(); @@ -2800,6 +2801,7 @@ static void __exit ipoib_cleanup_module(void) ib_sa_unregister_client(&ipoib_sa_client); ipoib_unregister_debugfs(); destroy_workqueue(ipoib_workqueue); + rcu_barrier(); } module_init(ipoib_init_module); diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c index 1015a51f750af4..df5d8295a7fc12 100644 --- a/drivers/infiniband/ulp/isert/ib_isert.c +++ b/drivers/infiniband/ulp/isert/ib_isert.c @@ -59,6 +59,8 @@ static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc); static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc); static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc); static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc); +static void isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, + struct ib_device *ib_dev); static int isert_sg_tablesize_set(const char *val, const struct kernel_param *kp) { @@ -496,6 +498,8 @@ isert_connect_release(struct isert_conn *isert_conn) if (isert_conn->qp) isert_destroy_qp(isert_conn); + isert_unmap_tx_desc(&isert_conn->login_tx_desc, device->ib_device); + if (isert_conn->login_desc) isert_free_login_buf(isert_conn); @@ -943,27 +947,16 @@ isert_put_login_tx(struct iscsit_conn *conn, struct iscsi_login *login, } if (!login->login_failed) { if (login->login_complete) { - ret = isert_alloc_rx_descriptors(isert_conn); - if (ret) - return ret; - - ret = isert_post_recvm(isert_conn, - ISERT_QP_MAX_RECV_DTOS); - if (ret) - return ret; - - /* Now we are in FULL_FEATURE phase */ - mutex_lock(&isert_conn->mutex); - isert_conn->state = ISER_CONN_FULL_FEATURE; - mutex_unlock(&isert_conn->mutex); - goto post_send; + /* Posted and sent from isert_get_rx_pdu(). */ + isert_conn->login_rsp_pending = true; + return 0; } ret = isert_login_post_recv(isert_conn); if (ret) return ret; } -post_send: + ret = isert_login_post_send(isert_conn, tx_desc); if (ret) return ret; @@ -971,6 +964,21 @@ post_send: return 0; } +static int +isert_check_login_req(struct isert_conn *isert_conn) +{ + struct iscsi_hdr *hdr = isert_get_iscsi_hdr(isert_conn->login_desc); + u32 dlength = ntoh24(hdr->dlength); + + if (unlikely(dlength > (u32)isert_conn->login_req_len)) { + isert_dbg("login PDU declares %u data bytes but only %d were received\n", + dlength, isert_conn->login_req_len); + return -EINVAL; + } + + return 0; +} + static void isert_rx_login_req(struct isert_conn *isert_conn) { @@ -1333,6 +1341,21 @@ isert_recv_done(struct ib_cq *cq, struct ib_wc *wc) ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr, ISER_RX_SIZE, DMA_FROM_DEVICE); + /* + * The data segment length declared in the BHS is attacker controlled + * and is used further down to read that many bytes out of the fixed + * size receive descriptor, so it has to be checked against the number + * of bytes that were actually received. Comparing without subtracting + * also rejects PDUs shorter than the iSER and iSCSI headers, which + * would otherwise be parsed out of stale descriptor contents. + */ + if (unlikely(wc->byte_len < ISER_HEADERS_LEN + ntoh24(hdr->dlength))) { + isert_err("PDU declares %u data bytes but only %u bytes were received\n", + ntoh24(hdr->dlength), wc->byte_len); + iscsit_cause_connection_reinstatement(isert_conn->conn, 0); + return; + } + isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags, (int)(wc->byte_len - ISER_HEADERS_LEN)); @@ -1394,8 +1417,12 @@ isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc) if (isert_conn->conn) { struct iscsi_login *login = isert_conn->conn->conn_login; - if (login && !login->first_request) + if (login && !login->first_request) { + if (isert_check_login_req(isert_conn)) + return; + isert_rx_login_req(isert_conn); + } } mutex_lock(&isert_conn->mutex); @@ -2360,6 +2387,10 @@ isert_get_login_rx(struct iscsit_conn *conn, struct iscsi_login *login) if (!login->first_request) return 0; + ret = isert_check_login_req(isert_conn); + if (ret) + return ret; + isert_rx_login_req(isert_conn); isert_info("before login_comp conn: %p\n", conn); @@ -2585,8 +2616,30 @@ static void isert_free_conn(struct iscsit_conn *conn) static void isert_get_rx_pdu(struct iscsit_conn *conn) { + struct isert_conn *isert_conn = conn->context; struct completion comp; + /* The login timeout timer can fail the login after isert_put_login_tx(). */ + if (!isert_conn->login_rsp_pending) + return; + + isert_conn->login_rsp_pending = false; + + /* The session is registered by now; see isert_put_login_tx(). */ + if (isert_alloc_rx_descriptors(isert_conn)) + return; + + if (isert_post_recvm(isert_conn, ISERT_QP_MAX_RECV_DTOS)) + return; + + /* Now we are in FULL_FEATURE phase */ + mutex_lock(&isert_conn->mutex); + isert_conn->state = ISER_CONN_FULL_FEATURE; + mutex_unlock(&isert_conn->mutex); + + if (isert_login_post_send(isert_conn, &isert_conn->login_tx_desc)) + return; + init_completion(&comp); wait_for_completion_interruptible(&comp); diff --git a/drivers/infiniband/ulp/isert/ib_isert.h b/drivers/infiniband/ulp/isert/ib_isert.h index 0b2dfd6e7e2707..0bac5aa66c8028 100644 --- a/drivers/infiniband/ulp/isert/ib_isert.h +++ b/drivers/infiniband/ulp/isert/ib_isert.h @@ -178,6 +178,7 @@ struct isert_conn { struct completion login_comp; struct completion login_req_comp; struct iser_tx_desc login_tx_desc; + bool login_rsp_pending; struct rdma_cm_id *cm_id; struct ib_qp *qp; struct ib_cq *cq; diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c index acbd787de265b5..93dcd15682d048 100644 --- a/drivers/infiniband/ulp/srp/ib_srp.c +++ b/drivers/infiniband/ulp/srp/ib_srp.c @@ -1945,7 +1945,8 @@ static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp, ch->req_lim += be32_to_cpu(rsp->req_lim_delta); if (rsp->tag == ch->tsk_mgmt_tag) { ch->tsk_mgmt_status = -1; - if (be32_to_cpu(rsp->resp_data_len) >= 4) + if (be32_to_cpu(rsp->resp_data_len) >= 4 && + byte_len >= sizeof(*rsp) + 4) ch->tsk_mgmt_status = rsp->data[3]; complete(&ch->tsk_mgmt_done); } else { @@ -2045,13 +2046,20 @@ static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta, } static void srp_process_cred_req(struct srp_rdma_ch *ch, - struct srp_cred_req *req) + struct srp_cred_req *req, u32 byte_len) { - struct srp_cred_rsp rsp = { - .opcode = SRP_CRED_RSP, - .tag = req->tag, - }; - s32 delta = be32_to_cpu(req->req_lim_delta); + struct srp_cred_rsp rsp = { .opcode = SRP_CRED_RSP }; + s32 delta; + + if (byte_len < sizeof(*req)) { + shost_printk(KERN_ERR, ch->target->scsi_host, PFX + "dropping truncated SRP_CRED_REQ (%u bytes received, %zu expected)\n", + byte_len, sizeof(*req)); + return; + } + + rsp.tag = req->tag; + delta = be32_to_cpu(req->req_lim_delta); if (srp_response_common(ch, delta, &rsp, sizeof(rsp))) shost_printk(KERN_ERR, ch->target->scsi_host, PFX @@ -2059,14 +2067,21 @@ static void srp_process_cred_req(struct srp_rdma_ch *ch, } static void srp_process_aer_req(struct srp_rdma_ch *ch, - struct srp_aer_req *req) + struct srp_aer_req *req, u32 byte_len) { struct srp_target_port *target = ch->target; - struct srp_aer_rsp rsp = { - .opcode = SRP_AER_RSP, - .tag = req->tag, - }; - s32 delta = be32_to_cpu(req->req_lim_delta); + struct srp_aer_rsp rsp = { .opcode = SRP_AER_RSP }; + s32 delta; + + if (byte_len < sizeof(*req)) { + shost_printk(KERN_ERR, target->scsi_host, PFX + "dropping truncated SRP_AER_REQ (%u bytes received, %zu expected)\n", + byte_len, sizeof(*req)); + return; + } + + rsp.tag = req->tag; + delta = be32_to_cpu(req->req_lim_delta); shost_printk(KERN_ERR, target->scsi_host, PFX "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun)); @@ -2108,11 +2123,11 @@ static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc) break; case SRP_CRED_REQ: - srp_process_cred_req(ch, iu->buf); + srp_process_cred_req(ch, iu->buf, wc->byte_len); break; case SRP_AER_REQ: - srp_process_aer_req(ch, iu->buf); + srp_process_aer_req(ch, iu->buf, wc->byte_len); break; case SRP_T_LOGOUT: diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c index f66cfd70c2636a..3ec42da1456b8d 100644 --- a/drivers/infiniband/ulp/srpt/ib_srpt.c +++ b/drivers/infiniband/ulp/srpt/ib_srpt.c @@ -960,6 +960,7 @@ static int srpt_alloc_rw_ctxs(struct srpt_send_ioctx *ioctx, struct srpt_rdma_ch *ch = ioctx->ch; struct scatterlist *prev = NULL; unsigned prev_nents; + u8 n_rdma, n_rw_ctx; int ret, i; if (nbufs == 1) { @@ -970,6 +971,9 @@ static int srpt_alloc_rw_ctxs(struct srpt_send_ioctx *ioctx, return -ENOMEM; } + n_rw_ctx = ioctx->n_rw_ctx; + n_rdma = ioctx->n_rdma; + for (i = ioctx->n_rw_ctx; i < nbufs; i++, db++) { struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i]; u64 remote_addr = be64_to_cpu(db->va); @@ -1016,6 +1020,9 @@ unwind: } if (ioctx->rw_ctxs != &ioctx->s_rw_ctx) kfree(ioctx->rw_ctxs); + ioctx->rw_ctxs = NULL; + ioctx->n_rw_ctx = n_rw_ctx; + ioctx->n_rdma = n_rdma; return ret; } @@ -1596,7 +1603,7 @@ static void srpt_handle_cmd(struct srpt_rdma_ch *ch, rc = target_init_cmd(cmd, ch->sess, &send_ioctx->sense_data[0], scsilun_to_int(&srp_cmd->lun), data_len, - TCM_SIMPLE_TAG, dir, TARGET_SCF_ACK_KREF); + cmd->sam_task_attr, dir, TARGET_SCF_ACK_KREF); if (rc != 0) { pr_debug("target_submit_cmd() returned %d for tag %#llx\n", rc, srp_cmd->tag); diff --git a/drivers/iommu/amd/debugfs.c b/drivers/iommu/amd/debugfs.c index 4c53b63613148e..5c573ec8e27a48 100644 --- a/drivers/iommu/amd/debugfs.c +++ b/drivers/iommu/amd/debugfs.c @@ -176,19 +176,13 @@ static ssize_t devid_write(struct file *filp, const char __user *ubuf, kfree(srcid_ptr); return -ENODEV; } - break; - } - - if (pci_seg->id != seg) { + sbdf = PCI_SEG_DEVID_TO_SBDF(seg, devid); kfree(srcid_ptr); - return -EINVAL; + return cnt; } - sbdf = PCI_SEG_DEVID_TO_SBDF(seg, devid); - kfree(srcid_ptr); - - return cnt; + return -EINVAL; } static int devid_show(struct seq_file *m, void *unused) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index e7d7b4cb9337fc..2563ebe9f24611 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -3091,14 +3091,28 @@ static void __init free_iommu_resources(void) /* SB IOAPIC for Hygon family 18h model 4h is on the device 0xb */ #define IOAPIC_SB_DEVID_FAM18H_M4H ((0x00 << 8) | PCI_DEVFN(0xb, 0)) +/* + * The Southbridge IOAPIC is assigned a GSI Base of 0 (handling interrupts + * 0 through 23). + */ +static int __init get_sb_ioapic_id(void) +{ + int idx = mp_find_ioapic(0); + + if (idx < 0) + return -ENODEV; + + return mpc_ioapic_id(idx); +} + static bool __init check_ioapic_information(void) { const char *fw_bug = FW_BUG; bool ret, has_sb_ioapic; - int idx; + int idx, sb_apicid; has_sb_ioapic = false; - ret = false; + ret = true; /* * If we have map overrides on the kernel command line the @@ -3108,6 +3122,16 @@ static bool __init check_ioapic_information(void) if (cmdline_maps) fw_bug = ""; + sb_apicid = get_sb_ioapic_id(); + if (sb_apicid < 0) { + /* + * Lack of SB IOAPIC registration is not a firmware bug, + * e.g. kernel booted with noapic or noacpi. + */ + fw_bug = ""; + goto out; + } + for (idx = 0; idx < nr_ioapics; idx++) { int devid, id = mpc_ioapic_id(idx); @@ -3116,17 +3140,16 @@ static bool __init check_ioapic_information(void) pr_err("%s: IOAPIC[%d] not in IVRS table\n", fw_bug, id); ret = false; - } else if (devid == IOAPIC_SB_DEVID || + } else if (id == sb_apicid && (devid == IOAPIC_SB_DEVID || (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON && boot_cpu_data.x86 == 0x18 && boot_cpu_data.x86_model >= 0x4 && boot_cpu_data.x86_model <= 0xf && - devid == IOAPIC_SB_DEVID_FAM18H_M4H)) { + devid == IOAPIC_SB_DEVID_FAM18H_M4H))) { has_sb_ioapic = true; - ret = true; } } - +out: if (!has_sb_ioapic) { /* * We expect the SB IOAPIC to be listed in the IVRS @@ -3137,6 +3160,7 @@ static bool __init check_ioapic_information(void) * device id for the IOAPIC in the system. */ pr_err("%s: No southbridge IOAPIC found\n", fw_bug); + ret = false; } if (!ret) diff --git a/drivers/iommu/amd/ppr.c b/drivers/iommu/amd/ppr.c index 80369ca1e316ef..1dde19a79e80f9 100644 --- a/drivers/iommu/amd/ppr.c +++ b/drivers/iommu/amd/ppr.c @@ -140,7 +140,7 @@ static void iommu_call_iopf_notifier(struct amd_iommu *iommu, u64 *raw) if (event.fault.prm.pasid == 0 || event.fault.prm.pasid >= dev_data->max_pasids) { pr_info_ratelimited("Invalid PASID : 0x%x, device : 0x%x\n", - event.fault.prm.pasid, pdev->dev.id); + event.fault.prm.pasid, dev_data->devid); goto out; } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 5f933d806a1478..35b7b2fd4a122f 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1240,9 +1240,9 @@ VISIBLE_IF_KUNIT void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target, __le64 *safe_bits) { - const __le64 eats_s1chk = + const u64 eats_s1chk = FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_S1CHK); - const __le64 eats_trans = + const u64 eats_trans = FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_TRANS); /* @@ -4553,8 +4553,9 @@ static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val, u32 reg; writel_relaxed(val, smmu->base + reg_off); - return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val, - 1, ARM_SMMU_POLL_TIMEOUT_US); + return readl_relaxed_poll_timeout_atomic(smmu->base + ack_off, reg, + reg == val, 1, + ARM_SMMU_POLL_TIMEOUT_US); } /* GBPA is "special" */ diff --git a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c index e5f52c0f6e9adf..ef08367567d395 100644 --- a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c +++ b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c @@ -320,16 +320,30 @@ static void tegra241_vintf0_handle_error(struct tegra241_vintf *vintf) while (map) { unsigned long lidx = __ffs64(map); - struct tegra241_vcmdq *vcmdq = vintf->lvcmdqs[lidx]; - u32 gerror = readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR)); + struct tegra241_vcmdq *vcmdq; + u32 gerror; + map &= ~BIT_ULL(lidx); + + /* Pairs with smp_store_release() publishing it */ + vcmdq = smp_load_acquire(&vintf->lvcmdqs[lidx]); + if (!vcmdq) + continue; + + gerror = readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR)); __arm_smmu_cmdq_skip_err(&vintf->cmdqv->smmu, &vcmdq->cmdq); writel(gerror, REG_VCMDQ_PAGE0(vcmdq, GERRORN)); - map &= ~BIT_ULL(lidx); } } } +/* + * The CMDQV error interrupt is edge-triggered, so a pending VINTF error fires + * this ISR once and does not re-assert. An unacked guest therefore cannot + * storm the host. The HW latches and forwards each new error event on its + * own, so an already-set ERR_MAP bit does not suppress the interrupt for a + * new error. + */ static irqreturn_t tegra241_cmdqv_isr(int irq, void *devid) { struct tegra241_cmdqv *cmdqv = (struct tegra241_cmdqv *)devid; @@ -352,16 +366,27 @@ static irqreturn_t tegra241_cmdqv_isr(int irq, void *devid) /* Handle VINTF0 and its LVCMDQs */ if (vintf_map & BIT_ULL(0)) { - tegra241_vintf0_handle_error(cmdqv->vintfs[0]); + struct tegra241_vintf *vintf0; + vintf_map &= ~BIT_ULL(0); + + /* NULL until tegra241_cmdqv_init_structures() publishes it */ + vintf0 = smp_load_acquire(&cmdqv->vintfs[0]); + if (vintf0) + tegra241_vintf0_handle_error(vintf0); } /* Handle other user VINTFs and their LVCMDQs */ while (vintf_map) { unsigned long idx = __ffs64(vintf_map); + struct tegra241_vintf *vintf; - tegra241_vintf_user_handle_error(cmdqv->vintfs[idx]); vintf_map &= ~BIT_ULL(idx); + + /* The slot may be published or torn down (NULL'd) concurrently */ + vintf = smp_load_acquire(&cmdqv->vintfs[idx]); + if (vintf) + tegra241_vintf_user_handle_error(vintf); } return IRQ_HANDLED; @@ -668,7 +693,6 @@ static int tegra241_vintf_init_lvcmdq(struct tegra241_vintf *vintf, u16 lidx, vcmdq->page0 = cmdqv->base + TEGRA241_VINTFi_LVCMDQ_PAGE0(idx, lidx); vcmdq->page1 = cmdqv->base + TEGRA241_VINTFi_LVCMDQ_PAGE1(idx, lidx); - vintf->lvcmdqs[lidx] = vcmdq; return 0; } @@ -684,7 +708,7 @@ static void tegra241_vintf_free_lvcmdq(struct tegra241_vintf *vintf, u16 lidx) dev_dbg(vintf->cmdqv->dev, "%sdeallocated\n", lvcmdq_error_header(vcmdq, header, 64)); /* Guest-owned VCMDQ is free-ed with hw_queue by iommufd core */ - if (vcmdq->vintf->hyp_own) + if (!vcmdq->vintf->idx) kfree(vcmdq); } @@ -707,14 +731,15 @@ tegra241_vintf_alloc_lvcmdq(struct tegra241_vintf *vintf, u16 lidx) /* Build an arm_smmu_cmdq for each LVCMDQ */ ret = tegra241_vcmdq_alloc_smmu_cmdq(vcmdq); if (ret) - goto deinit_lvcmdq; + goto free_vcmdq; + + /* Pairs with the smp_load_acquire() in the error ISR */ + smp_store_release(&vintf->lvcmdqs[lidx], vcmdq); dev_dbg(cmdqv->dev, "%sallocated\n", lvcmdq_error_header(vcmdq, header, 64)); return vcmdq; -deinit_lvcmdq: - tegra241_vintf_deinit_lvcmdq(vintf, lidx); free_vcmdq: kfree(vcmdq); return ERR_PTR(ret); @@ -725,8 +750,18 @@ free_vcmdq: static void tegra241_cmdqv_deinit_vintf(struct tegra241_cmdqv *cmdqv, u16 idx) { kfree(cmdqv->vintfs[idx]->lvcmdqs); + /* + * Clear the slot and drain any in-flight ISR before returning idx to + * the IDA, so a concurrent create that reuses idx cannot have its + * freshly published VINTF erased here. A plain WRITE_ONCE() suffices + * since clearing the slot publishes no data. This also covers the + * init-failure unwind, which reaches deinit_vintf() without the + * destroy callback. + */ + WRITE_ONCE(cmdqv->vintfs[idx], NULL); + if (cmdqv->irq > 0) + synchronize_irq(cmdqv->irq); ida_free(&cmdqv->vintf_ids, idx); - cmdqv->vintfs[idx] = NULL; } static int tegra241_cmdqv_init_vintf(struct tegra241_cmdqv *cmdqv, u16 max_idx, @@ -752,7 +787,8 @@ static int tegra241_cmdqv_init_vintf(struct tegra241_cmdqv *cmdqv, u16 max_idx, return -ENOMEM; } - cmdqv->vintfs[idx] = vintf; + /* Pairs with the smp_load_acquire() in tegra241_cmdqv_isr() */ + smp_store_release(&cmdqv->vintfs[idx], vintf); return ret; } @@ -770,7 +806,7 @@ static void tegra241_cmdqv_remove_vintf(struct tegra241_cmdqv *cmdqv, u16 idx) dev_dbg(cmdqv->dev, "VINTF%u: deallocated\n", vintf->idx); tegra241_cmdqv_deinit_vintf(cmdqv, idx); - if (!vintf->hyp_own) { + if (vintf->idx) { mutex_destroy(&vintf->lvcmdq_mutex); ida_destroy(&vintf->sids); /* Guest-owned VINTF is free-ed with viommu by iommufd core */ @@ -796,6 +832,14 @@ static void tegra241_cmdqv_remove(struct arm_smmu_device *smmu) container_of(smmu, struct tegra241_cmdqv, smmu); u16 idx; + /* + * Free the IRQ before tearing down the VINTFs. free_irq() waits for any + * in-flight tegra241_cmdqv_isr() to finish and blocks new ones, so the + * ISR cannot dereference a VINTF that is freed by the loop below. + */ + if (cmdqv->irq > 0) + free_irq(cmdqv->irq, cmdqv); + /* Remove VINTF resources */ for (idx = 0; idx < cmdqv->num_vintfs; idx++) { if (cmdqv->vintfs[idx]) { @@ -808,8 +852,6 @@ static void tegra241_cmdqv_remove(struct arm_smmu_device *smmu) /* Remove cmdqv resources */ ida_destroy(&cmdqv->vintf_ids); - if (cmdqv->irq > 0) - free_irq(cmdqv->irq, cmdqv); iounmap(cmdqv->base); kfree(cmdqv->vintfs); put_device(cmdqv->dev); /* smmu->impl_dev */ @@ -881,6 +923,12 @@ static int tegra241_cmdqv_init_structures(struct arm_smmu_device *smmu) ret = tegra241_cmdqv_init_vintf(cmdqv, 0, vintf); if (ret) { dev_err(cmdqv->dev, "failed to init vintf0: %d\n", ret); + /* + * tegra241_cmdqv_init_vintf() failed to publish the vintf0 to + * cmdqv->vintfs[], so the probe unwind path that goes through + * cmdqv->vintfs[] would miss it. Free it here. + */ + kfree(vintf); return ret; } @@ -902,16 +950,22 @@ static int tegra241_cmdqv_init_structures(struct arm_smmu_device *smmu) static struct dentry *cmdqv_debugfs_dir; #endif -static struct arm_smmu_device * -__tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res, - int irq) +/* + * Probe the CMDQV and reallocate @smmu into the larger cmdqv->smmu. + * + * devm_krealloc() may relocate and free the original @smmu, so update *smmu to + * the new pointer once it succeeds. The error paths after it do the same, so a + * caller falling back keeps a live @smmu instead of the freed original. + */ +static int __tegra241_cmdqv_probe(struct arm_smmu_device **smmu, + struct resource *res, int irq) { static const struct arm_smmu_impl_ops init_ops = { .init_structures = tegra241_cmdqv_init_structures, .device_remove = tegra241_cmdqv_remove, }; - struct tegra241_cmdqv *cmdqv = NULL; - struct arm_smmu_device *new_smmu; + struct device *dev = (*smmu)->dev; + struct tegra241_cmdqv *cmdqv; void __iomem *base; u32 regval; int ret; @@ -920,38 +974,30 @@ __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res, base = ioremap(res->start, resource_size(res)); if (!base) { - dev_err(smmu->dev, "failed to ioremap\n"); - return NULL; + dev_err(dev, "failed to ioremap\n"); + return -ENOMEM; } regval = readl(base + TEGRA241_CMDQV_CONFIG); if (disable_cmdqv) { - dev_info(smmu->dev, "Detected disable_cmdqv=true\n"); + dev_info(dev, "Detected disable_cmdqv=true\n"); writel(regval & ~CMDQV_EN, base + TEGRA241_CMDQV_CONFIG); + ret = -ENODEV; goto iounmap; } - cmdqv = devm_krealloc(smmu->dev, smmu, sizeof(*cmdqv), GFP_KERNEL); - if (!cmdqv) + cmdqv = devm_krealloc(dev, *smmu, sizeof(*cmdqv), GFP_KERNEL); + if (!cmdqv) { + ret = -ENOMEM; goto iounmap; - new_smmu = &cmdqv->smmu; + } + *smmu = &cmdqv->smmu; cmdqv->irq = irq; cmdqv->base = base; - cmdqv->dev = smmu->impl_dev; + cmdqv->dev = (*smmu)->impl_dev; cmdqv->base_phys = res->start; - if (cmdqv->irq > 0) { - ret = request_threaded_irq(irq, NULL, tegra241_cmdqv_isr, - IRQF_ONESHOT, "tegra241-cmdqv", - cmdqv); - if (ret) { - dev_err(cmdqv->dev, "failed to request irq (%d): %d\n", - cmdqv->irq, ret); - goto iounmap; - } - } - regval = readl_relaxed(REG_CMDQV(cmdqv, PARAM)); cmdqv->num_vintfs = 1 << FIELD_GET(CMDQV_NUM_VINTF_LOG2, regval); cmdqv->num_vcmdqs = 1 << FIELD_GET(CMDQV_NUM_VCMDQ_LOG2, regval); @@ -961,11 +1007,28 @@ __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res, cmdqv->vintfs = kzalloc_objs(*cmdqv->vintfs, cmdqv->num_vintfs); - if (!cmdqv->vintfs) - goto free_irq; + if (!cmdqv->vintfs) { + ret = -ENOMEM; + goto iounmap; + } ida_init(&cmdqv->vintf_ids); + /* + * Request the IRQ only after cmdqv->vintfs is allocated and zeroed, so + * the ISR would not walk an uninitialized array. + */ + if (cmdqv->irq > 0) { + ret = request_threaded_irq(irq, NULL, tegra241_cmdqv_isr, + IRQF_ONESHOT, "tegra241-cmdqv", + cmdqv); + if (ret) { + dev_err(cmdqv->dev, "failed to request irq (%d): %d\n", + cmdqv->irq, ret); + goto free_vintfs; + } + } + #ifdef CONFIG_IOMMU_DEBUGFS if (!cmdqv_debugfs_dir) { cmdqv_debugfs_dir = @@ -976,24 +1039,23 @@ __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res, #endif /* Provide init-level ops only, until tegra241_cmdqv_init_structures */ - new_smmu->impl_ops = &init_ops; + cmdqv->smmu.impl_ops = &init_ops; - return new_smmu; + return 0; -free_irq: - if (cmdqv->irq > 0) - free_irq(cmdqv->irq, cmdqv); +free_vintfs: + ida_destroy(&cmdqv->vintf_ids); + kfree(cmdqv->vintfs); iounmap: iounmap(base); - return NULL; + return ret; } struct arm_smmu_device *tegra241_cmdqv_probe(struct arm_smmu_device *smmu) { struct platform_device *pdev = to_platform_device(smmu->impl_dev); - struct arm_smmu_device *new_smmu; struct resource *res; - int irq; + int irq, ret; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!res) { @@ -1006,15 +1068,15 @@ struct arm_smmu_device *tegra241_cmdqv_probe(struct arm_smmu_device *smmu) dev_warn(&pdev->dev, "no interrupt. errors will not be reported\n"); - new_smmu = __tegra241_cmdqv_probe(smmu, res, irq); - if (new_smmu) - return new_smmu; + ret = __tegra241_cmdqv_probe(&smmu, res, irq); + if (!ret) + return smmu; out_fallback: dev_info(smmu->impl_dev, "Falling back to standard SMMU CMDQ\n"); smmu->options &= ~ARM_SMMU_OPT_TEGRA241_CMDQV; put_device(smmu->impl_dev); - return ERR_PTR(-ENODEV); + return smmu; } /* User space VINTF and VCMDQ Functions */ @@ -1142,13 +1204,15 @@ static int tegra241_vintf_alloc_lvcmdq_user(struct iommufd_hw_queue *hw_queue, if (ret) goto unmap_lvcmdq; + /* No lockless reader of a user VINTF's lvcmdqs[]; mutex-serialized */ + vintf->lvcmdqs[lidx] = vcmdq; + hw_queue->destroy = &tegra241_vintf_destroy_lvcmdq_user; mutex_unlock(&vintf->lvcmdq_mutex); return 0; unmap_lvcmdq: tegra241_vcmdq_unmap_lvcmdq(vcmdq); - tegra241_vintf_deinit_lvcmdq(vintf, lidx); undepend_vcmdq: if (vcmdq->prev) iommufd_hw_queue_undepend(vcmdq, vcmdq->prev, core); @@ -1194,7 +1258,8 @@ static int tegra241_vintf_init_vsid(struct iommufd_vdevice *vdev) if (virt_sid > FIELD_MAX(VINTF_SID_MATCH_VIRT_SID)) return -EINVAL; - WARN_ON_ONCE(master->num_streams != 1); + if (master->num_streams != 1) + return -EOPNOTSUPP; /* Find an empty pair of SID_REPLACE and SID_MATCH */ sidx = ida_alloc_max(&vintf->sids, vintf->cmdqv->num_sids_per_vintf - 1, diff --git a/drivers/iommu/arm/arm-smmu/qcom_iommu.c b/drivers/iommu/arm/arm-smmu/qcom_iommu.c index a1e8cf29f59489..09f2ee6be988b2 100644 --- a/drivers/iommu/arm/arm-smmu/qcom_iommu.c +++ b/drivers/iommu/arm/arm-smmu/qcom_iommu.c @@ -200,7 +200,7 @@ static irqreturn_t qcom_iommu_fault(int irq, void *dev) fsynr = iommu_readl(ctx, ARM_SMMU_CB_FSYNR0); iova = iommu_readq(ctx, ARM_SMMU_CB_FAR); - if (!report_iommu_fault(ctx->domain, ctx->dev, iova, 0)) { + if (report_iommu_fault(ctx->domain, ctx->dev, iova, 0)) { dev_err_ratelimited(ctx->dev, "Unhandled context fault: fsr=0x%x, " "iova=0x%016llx, fsynr=0x%x, cb=%d\n", @@ -855,7 +855,7 @@ static int qcom_iommu_device_probe(struct platform_device *pdev) ret = iommu_device_register(&qcom_iommu->iommu, &qcom_iommu_ops, dev); if (ret) { dev_err(dev, "Failed to register iommu\n"); - goto err_pm_disable; + goto err_sysfs_remove; } if (qcom_iommu->local_base) { @@ -866,6 +866,8 @@ static int qcom_iommu_device_probe(struct platform_device *pdev) return 0; +err_sysfs_remove: + iommu_device_sysfs_remove(&qcom_iommu->iommu); err_pm_disable: pm_runtime_disable(dev); return ret; diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 9abaec0703efb4..ae814d8c99c04a 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1671,13 +1671,16 @@ void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, } if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && - !gfpflags_allow_blocking(gfp) && !coherent) + !gfpflags_allow_blocking(gfp) && !coherent) { page = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr, - gfp, NULL); - else + gfp, NULL); + if (!page) + return NULL; + } else { cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs); - if (!cpu_addr) - return NULL; + if (!cpu_addr) + return NULL; + } *handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot, dev->coherent_dma_mask); @@ -2204,6 +2207,19 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, dma_addr_t iova; int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; size_t size = cookie_msi_granule(domain); + static DEFINE_MUTEX(msi_prepare_lock); + + /* + * Normally a device's default domain is only ever attached to that + * device's own group, and the group mutex held by + * iommu_group_mutex_assert()'s callers is enough on its own. A VFIO + * type1 container is the one case that breaks that assumption: it + * can merge devices from different groups onto one domain, so two + * devices' group mutexes don't serialize each other here. A static + * lock is sufficient due to the expectation that this is a corner + * case that will never be contended in practice. + */ + guard(mutex)(&msi_prepare_lock); msi_addr &= ~(phys_addr_t)(size - 1); list_for_each_entry(msi_page, msi_page_list, list) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 4e7ba60f3a0a22..9a87ac1250d7b5 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1108,7 +1108,8 @@ static void copied_context_tear_down(struct intel_iommu *iommu, assert_spin_locked(&iommu->lock); did_old = context_domain_id(context); - context_clear_entry(context); + context_clear_present(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); if (did_old < cap_ndoms(iommu->cap)) { iommu->flush.flush_context(iommu, did_old, @@ -1119,6 +1120,9 @@ static void copied_context_tear_down(struct intel_iommu *iommu, DMA_TLB_DSI_FLUSH); } + context_clear_entry(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); + clear_context_copied(iommu, bus, devfn); } @@ -1246,7 +1250,7 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 context_clear_present(context); __iommu_flush_cache(iommu, context, sizeof(*context)); spin_unlock(&iommu->lock); - intel_context_flush_no_pasid(info, context, did); + intel_context_flush_no_pasid(info, context, did, PCI_DEVID(bus, devfn)); context_clear_entry(context); __iommu_flush_cache(iommu, context, sizeof(*context)); } @@ -1446,7 +1450,7 @@ static int copy_context_table(struct intel_iommu *iommu, struct context_entry **tbl, int bus, bool ext) { - int tbl_idx, pos = 0, idx, devfn, ret = 0, did; + int tbl_idx, tbl_slot = 0, idx, devfn, ret = 0, did; struct context_entry *new_ce = NULL, ce; struct context_entry *old_ce = NULL; struct root_entry re; @@ -1462,10 +1466,9 @@ static int copy_context_table(struct intel_iommu *iommu, if (idx == 0) { /* First save what we may have and clean up */ if (new_ce) { - tbl[tbl_idx] = new_ce; + tbl[tbl_idx + tbl_slot] = new_ce; __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE); - pos = 1; } if (old_ce) @@ -1487,6 +1490,9 @@ static int copy_context_table(struct intel_iommu *iommu, } } + /* Track if saving UCTP or LCTP entries in scalable mode */ + tbl_slot = ext && devfn >= 0x80 ? 1 : 0; + ret = -ENOMEM; old_ce = memremap(old_ce_phys, PAGE_SIZE, MEMREMAP_WB); @@ -1515,7 +1521,7 @@ static int copy_context_table(struct intel_iommu *iommu, new_ce[idx] = ce; } - tbl[tbl_idx + pos] = new_ce; + tbl[tbl_idx + tbl_slot] = new_ce; __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE); @@ -3135,13 +3141,13 @@ static int intel_iommu_attach_device(struct iommu_domain *domain, if (ret) return ret; - ret = iopf_for_domain_set(domain, dev); + ret = iopf_for_domain_replace(domain, old, dev); if (ret) return ret; ret = dmar_domain_attach_device(to_dmar_domain(domain), dev); if (ret) - iopf_for_domain_remove(domain, dev); + iopf_for_domain_replace(old, domain, dev); return ret; } @@ -3316,6 +3322,7 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev) return &iommu->iommu; free_table: + intel_pasid_teardown_sm_context(dev); intel_pasid_free_table(dev); clear_rbtree: device_rbtree_remove(info); @@ -3844,10 +3851,13 @@ static int identity_domain_attach_dev(struct iommu_domain *domain, return 0; /* - * No PRI support with the global identity domain. No need to enable or - * disable PRI in this path as the iommu has been put in the blocking - * state. + * The identity domain has no iopf_handler, so no IOPF reference is + * taken for it. The reference held by the old domain must still be + * released here; putting the device in the blocking state above does + * not affect the IOPF reference count. */ + iopf_for_domain_remove(old, dev); + if (sm_supported(iommu)) ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID); else diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 2cee36138d6e1e..2df59d999715ae 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -1240,7 +1240,7 @@ void cache_tag_flush_range_np(struct dmar_domain *domain, unsigned long start, unsigned long end); void intel_context_flush_no_pasid(struct device_domain_info *info, - struct context_entry *context, u16 did); + struct context_entry *context, u16 did, u16 sid); int intel_iommu_enable_prq(struct intel_iommu *iommu); int intel_iommu_finish_prq(struct intel_iommu *iommu); diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 81353fd46b37e1..e4f24d3f19a61a 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -751,7 +751,7 @@ static void device_pasid_table_teardown(struct device *dev, u8 bus, u8 devfn) context_clear_present(context); __iommu_flush_cache(iommu, context, sizeof(*context)); spin_unlock(&iommu->lock); - intel_context_flush_no_pasid(info, context, did); + intel_context_flush_no_pasid(info, context, did, PCI_DEVID(bus, devfn)); context_clear_entry(context); __iommu_flush_cache(iommu, context, sizeof(*context)); } @@ -955,9 +955,12 @@ static void __context_flush_dev_iotlb(struct device_domain_info *info) * This helper can only be used when IOMMU is working in the legacy mode or * IOMMU is in scalable mode but all PASID table entries of the device are * non-present. + * + * @sid identifies the context entry that was modified, which may be a DMA + * alias of @info->dev rather than its own requester ID. */ void intel_context_flush_no_pasid(struct device_domain_info *info, - struct context_entry *context, u16 did) + struct context_entry *context, u16 did, u16 sid) { struct intel_iommu *iommu = info->iommu; @@ -967,7 +970,7 @@ void intel_context_flush_no_pasid(struct device_domain_info *info, * when operating in scalable mode. Therefore the @did value doesn't * matter in scalable mode. */ - iommu->flush.flush_context(iommu, did, PCI_DEVID(info->bus, info->devfn), + iommu->flush.flush_context(iommu, did, sid, DMA_CCMD_MASK_NOBIT, DMA_CCMD_DEVICE_INVL); /* diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c index d488c23fd35383..c5d122f33ba2dc 100644 --- a/drivers/iommu/iommufd/device.c +++ b/drivers/iommu/iommufd/device.c @@ -148,29 +148,22 @@ static void iommufd_device_remove_vdev(struct iommufd_device *idev) if (!idev->vdev) goto out_unlock; - vdev = iommufd_get_vdevice(idev->ictx, idev->vdev->obj.id); + vdev = idev->vdev; + /* * An ongoing vdev destroy ioctl has removed the vdev from the object * xarray, but has not finished iommufd_vdevice_destroy() yet as it * needs the same mutex. We exit the locking then wait on wait_cnt * reference for the vdev destruction. */ - if (IS_ERR(vdev)) - goto out_unlock; - - /* Should never happen */ - if (WARN_ON(vdev != idev->vdev)) { - iommufd_put_object(idev->ictx, &vdev->obj); + if (iommufd_try_inc_users(idev->ictx, &vdev->obj)) goto out_unlock; - } /* * vdev is still alive. Hold a users refcount to prevent racing with * userspace destruction, then use iommufd_object_tombstone_user() to * destroy it and leave a tombstone. */ - refcount_inc(&vdev->obj.users); - iommufd_put_object(idev->ictx, &vdev->obj); mutex_unlock(&idev->igroup->lock); iommufd_object_tombstone_user(idev->ictx, &vdev->obj); return; diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h index 43fbc5bed8de3f..421d0cc7c1bc3a 100644 --- a/drivers/iommu/iommufd/iommufd_private.h +++ b/drivers/iommu/iommufd/iommufd_private.h @@ -182,6 +182,7 @@ static inline bool iommufd_lock_obj(struct iommufd_object *obj) return true; } +int iommufd_try_inc_users(struct iommufd_ctx *ictx, struct iommufd_object *obj); struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id, enum iommufd_object_type type); static inline void iommufd_put_object(struct iommufd_ctx *ictx, @@ -698,14 +699,6 @@ void iommufd_vdevice_abort(struct iommufd_object *obj); int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd); void iommufd_hw_queue_destroy(struct iommufd_object *obj); -static inline struct iommufd_vdevice * -iommufd_get_vdevice(struct iommufd_ctx *ictx, u32 id) -{ - return container_of(iommufd_get_object(ictx, id, - IOMMUFD_OBJ_VDEVICE), - struct iommufd_vdevice, obj); -} - #ifdef CONFIG_IOMMUFD_TEST int iommufd_test(struct iommufd_ucmd *ucmd); void iommufd_selftest_destroy(struct iommufd_object *obj); diff --git a/drivers/iommu/iommufd/main.c b/drivers/iommu/iommufd/main.c index 8c6d43601afbed..e1097a1db21ad9 100644 --- a/drivers/iommu/iommufd/main.c +++ b/drivers/iommu/iommufd/main.c @@ -180,6 +180,26 @@ struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id, return obj; } +/* + * Increment the users count of an object outside the context of an ioctl that + * has already locked it. The users refcount cannot be increased on an already + * created object unless the object is installed in the xarray, otherwise things + * are racing with a parallel destruction. + */ +int iommufd_try_inc_users(struct iommufd_ctx *ictx, struct iommufd_object *obj) +{ + struct iommufd_object *cur; + + xa_lock(&ictx->objects); + cur = xa_load(&ictx->objects, obj->id); + if (cur == obj) + refcount_inc(&obj->users); + xa_unlock(&ictx->objects); + if (cur != obj) + return -EBUSY; + return 0; +} + static int iommufd_object_dec_wait(struct iommufd_ctx *ictx, struct iommufd_object *to_destroy) { diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c index 727b59799d5f08..55f21c799d8e67 100644 --- a/drivers/iommu/iommufd/selftest.c +++ b/drivers/iommu/iommufd/selftest.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -1705,6 +1706,9 @@ static int iommufd_test_dirty(struct iommufd_ucmd *ucmd, unsigned int mockpt_id, if (!page_size || !length || iova % page_size || length % page_size || !uptr) return -EINVAL; + max = length / page_size; + if (max > SZ_16M * BITS_PER_BYTE) + return -EOVERFLOW; hwpt = get_md_pagetable(ucmd, mockpt_id, &mock); if (IS_ERR(hwpt)) @@ -1715,7 +1719,6 @@ static int iommufd_test_dirty(struct iommufd_ucmd *ucmd, unsigned int mockpt_id, goto out_put; } - max = length / page_size; tmp = kvzalloc(DIV_ROUND_UP(max, BITS_PER_LONG) * sizeof(unsigned long), GFP_KERNEL_ACCOUNT); if (!tmp) { diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index d0d926be7495cf..9a3ed70563b1b2 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -720,7 +720,7 @@ static int msm_iommu_probe(struct platform_device *pdev) iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL); if (!iommu) - return -ENODEV; + return -ENOMEM; iommu->dev = &pdev->dev; INIT_LIST_HEAD(&iommu->ctx_list); diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index ac97dd2868d4b8..e907c99531423e 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -88,7 +88,7 @@ struct dma_iommu_mapping { /* MTK generation one iommu HW only support 4K size mapping */ #define MT2701_IOMMU_PAGE_SHIFT 12 #define MT2701_IOMMU_PAGE_SIZE (1UL << MT2701_IOMMU_PAGE_SHIFT) -#define MT2701_LARB_NR_MAX 3 +#define MT2701_LARB_NR_MAX 4 /* * MTK m4u support 4GB iova address space, and only support 4K page diff --git a/drivers/ipack/devices/ipoctal.c b/drivers/ipack/devices/ipoctal.c index 1bbefc6de708b5..bf71b8952a7c00 100644 --- a/drivers/ipack/devices/ipoctal.c +++ b/drivers/ipack/devices/ipoctal.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,8 @@ static const struct tty_operations ipoctal_fops; +static void ipoctal_release(struct kref *kref); + struct ipoctal_channel { struct ipoctal_stats stats; unsigned int nb_bytes; @@ -49,6 +52,9 @@ struct ipoctal { struct tty_driver *tty_drv; u8 __iomem *mem8_space; u8 __iomem *int_space; + struct kref kref; + struct module *carrier_owner; + bool removed; }; static inline struct ipoctal *chan_to_ipoctal(struct ipoctal_channel *chan, @@ -70,8 +76,14 @@ static void ipoctal_reset_channel(struct ipoctal_channel *channel) static int ipoctal_port_activate(struct tty_port *port, struct tty_struct *tty) { struct ipoctal_channel *channel; + struct ipoctal *ipoctal; channel = dev_get_drvdata(tty->dev); + ipoctal = chan_to_ipoctal(channel, tty->index); + + + if (ipoctal->removed) + return -ENODEV; /* * Enable RX. TX will be enabled when @@ -95,6 +107,7 @@ static int ipoctal_install(struct tty_driver *driver, struct tty_struct *tty) if (res) goto err_put_carrier; + kref_get(&ipoctal->kref); tty->driver_data = channel; return 0; @@ -460,8 +473,13 @@ static ssize_t ipoctal_write_tty(struct tty_struct *tty, const u8 *buf, size_t count) { struct ipoctal_channel *channel = tty->driver_data; + struct ipoctal *ipoctal = chan_to_ipoctal(channel, tty->index); size_t char_copied; + + if (ipoctal->removed || !channel->tty_port.xmit_buf) + return 0; + char_copied = ipoctal_copy_write_buffer(channel, buf, count); /* As the IP-OCTAL 485 only supports half duplex, do it manually */ @@ -501,8 +519,13 @@ static void ipoctal_set_termios(struct tty_struct *tty, unsigned char mr2 = 0; unsigned char csr = 0; struct ipoctal_channel *channel = tty->driver_data; + struct ipoctal *ipoctal = chan_to_ipoctal(channel, tty->index); speed_t baud; + + if (ipoctal->removed) + return; + cflag = tty->termios.c_cflag; /* Disable and reset everything before change the setup */ @@ -631,10 +654,16 @@ static void ipoctal_hangup(struct tty_struct *tty) { unsigned long flags; struct ipoctal_channel *channel = tty->driver_data; + struct ipoctal *ipoctal; if (channel == NULL) return; + ipoctal = chan_to_ipoctal(channel, tty->index); + + if (ipoctal->removed) + return; + spin_lock_irqsave(&channel->lock, flags); channel->nb_bytes = 0; channel->pointer_read = 0; @@ -651,10 +680,16 @@ static void ipoctal_hangup(struct tty_struct *tty) static void ipoctal_shutdown(struct tty_struct *tty) { struct ipoctal_channel *channel = tty->driver_data; + struct ipoctal *ipoctal; if (channel == NULL) return; + ipoctal = chan_to_ipoctal(channel, tty->index); + + if (ipoctal->removed) + return; + ipoctal_reset_channel(channel); tty_port_set_initialized(&channel->tty_port, false); } @@ -664,8 +699,9 @@ static void ipoctal_cleanup(struct tty_struct *tty) struct ipoctal_channel *channel = tty->driver_data; struct ipoctal *ipoctal = chan_to_ipoctal(channel, tty->index); - /* release the carrier driver */ - ipack_put_carrier(ipoctal->dev); + /* release the carrier driver via cached owner */ + module_put(ipoctal->carrier_owner); + kref_put(&ipoctal->kref, ipoctal_release); } static const struct tty_operations ipoctal_fops = { @@ -683,6 +719,13 @@ static const struct tty_operations ipoctal_fops = { .cleanup = ipoctal_cleanup, }; +static void ipoctal_release(struct kref *kref) +{ + struct ipoctal *ipoctal = container_of(kref, struct ipoctal, kref); + + kfree(ipoctal); +} + static int ipoctal_probe(struct ipack_device *dev) { int res; @@ -692,7 +735,10 @@ static int ipoctal_probe(struct ipack_device *dev) if (ipoctal == NULL) return -ENOMEM; + kref_init(&ipoctal->kref); + ipoctal->dev = dev; + ipoctal->carrier_owner = dev->bus->owner; res = ipoctal_inst_slot(ipoctal, dev->bus->bus_nr, dev->slot); if (res) goto out_uninst; @@ -701,7 +747,7 @@ static int ipoctal_probe(struct ipack_device *dev) return 0; out_uninst: - kfree(ipoctal); + kref_put(&ipoctal->kref, ipoctal_release); return res; } @@ -709,6 +755,8 @@ static void __ipoctal_remove(struct ipoctal *ipoctal) { int i; + ipoctal->removed = true; + ipoctal->dev->bus->ops->free_irq(ipoctal->dev); for (i = 0; i < NR_CHANNELS; i++) { @@ -725,7 +773,7 @@ static void __ipoctal_remove(struct ipoctal *ipoctal) tty_unregister_driver(ipoctal->tty_drv); kfree(ipoctal->tty_drv->name); tty_driver_kref_put(ipoctal->tty_drv); - kfree(ipoctal); + kref_put(&ipoctal->kref, ipoctal_release); } static void ipoctal_remove(struct ipack_device *idev) diff --git a/drivers/irqchip/irq-ast2700-intc0.c b/drivers/irqchip/irq-ast2700-intc0.c index 14b8b88f1179ba..1e375e9dffae1b 100644 --- a/drivers/irqchip/irq-ast2700-intc0.c +++ b/drivers/irqchip/irq-ast2700-intc0.c @@ -48,9 +48,9 @@ #define INTC0_INTMX_IER 0x1b00 #define INTC0_INTMX_ISR 0x1b04 #define INTC0_INTMX_BANK_SIZE 0x10 -#define INTC0_INTM_BANK_NUM 3 #define INTC0_IRQS_PER_BANK 32 #define INTM_IRQS_PER_BANK 10 +#define INTC0_INTM_BANK_NUM (INTC0_INTM_NUM / INTM_IRQS_PER_BANK) #define INTC0_SEL_BASE 0x200 #define INTC0_SEL_BANK_SIZE 0x4 #define INTC0_SEL_ROUTE_SIZE 0x100 @@ -312,7 +312,7 @@ int aspeed_intc0_resolve_route(const struct irq_domain *c0domain, size_t nc1outs return -ENOENT; if (!IS_ENABLED(CONFIG_ASPEED_AST2700_INTC_TEST) && - !fwnode_device_is_compatible(c0domain->fwnode, "aspeed,ast2700-intc0")) + !of_device_is_compatible(to_of_node(c0domain->fwnode), "aspeed,ast2700-intc0")) return -ENODEV; intc0 = c0domain->host_data; diff --git a/drivers/irqchip/irq-ast2700-intc1.c b/drivers/irqchip/irq-ast2700-intc1.c index 59e8f0d5ddcd29..df6acbafce2d01 100644 --- a/drivers/irqchip/irq-ast2700-intc1.c +++ b/drivers/irqchip/irq-ast2700-intc1.c @@ -157,7 +157,8 @@ static int aspeed_intc1_irq_domain_activate(struct irq_domain *domain, intc1->ranges.nranges, intc1->ranges.ranges, &resolved); if (rc < 0) { - if (!fwnode_device_is_compatible(intc1->upstream->fwnode, "riscv,aplic")) { + if (!of_device_is_compatible(to_of_node(intc1->upstream->fwnode), + "riscv,aplic")) { dev_warn(intc1->dev, "Failed to resolve interrupt route for hwirq %lu in domain %s\n", data->hwirq, domain->name); diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 6f5811aae59c17..274b9761c0eec4 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -4592,6 +4592,13 @@ static int its_vpe_init(struct its_vpe *vpe) static void its_vpe_teardown(struct its_vpe *vpe) { + /* + * If vpt_page is NULL, then its_vpe_init() has failed, and + * there is nothing to do as no resource has been allocated. + */ + if (vpe->vpt_page == NULL) + return; + its_vpe_db_proxy_unmap(vpe); its_vpe_id_free(vpe->vpe_id); its_free_pending_table(vpe->vpt_page); @@ -4672,8 +4679,10 @@ static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq irqd_set_resend_when_in_progress(irq_get_irq_data(virq + i)); } - if (err) + if (err) { + its_vpe_teardown(vm->vpes[i]); its_vpe_irq_domain_free(domain, virq, i); + } return err; } @@ -5320,7 +5329,7 @@ static int __init its_probe_one(struct its_node *its) err = its_init_domain(its); if (err) - goto out_free_tables; + goto out_free_collection; raw_spin_lock(&its_lock); list_add(&its->entry, &its_nodes); @@ -5328,6 +5337,8 @@ static int __init its_probe_one(struct its_node *its) return 0; +out_free_collection: + kfree(its->collections); out_free_tables: its_free_tables(its); out_free_cmd: @@ -5739,9 +5750,13 @@ static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header, its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE; err = its_probe_one(its); - if (!err) - return 0; + if (err) + goto probe_err; + + return 0; +probe_err: + its_node_destroy(its); node_err: iort_deregister_domain_token(its_entry->translation_id); dom_err: diff --git a/drivers/irqchip/irq-gic-v5-irs.c b/drivers/irqchip/irq-gic-v5-irs.c index f3fce0b1e25d96..b3feb6340b59e7 100644 --- a/drivers/irqchip/irq-gic-v5-irs.c +++ b/drivers/irqchip/irq-gic-v5-irs.c @@ -199,7 +199,7 @@ int gicv5_irs_iste_alloc(const u32 lpi) if (!gicv5_global_data.ist.l2) return 0; - irs_data = per_cpu(per_cpu_irs_data, smp_processor_id()); + irs_data = per_cpu(per_cpu_irs_data, 0); if (!irs_data) return -ENOENT; @@ -488,6 +488,12 @@ static int gicv5_irs_wait_for_idle(struct gicv5_irs_chip_data *irs_data) GICV5_IRS_CR0_IDLE, NULL); } +static void __init gicv5_irs_disable(struct gicv5_irs_chip_data *irs_data) +{ + irs_writel_relaxed(irs_data, 0, GICV5_IRS_CR0); + gicv5_irs_wait_for_idle(irs_data); +} + void gicv5_irs_syncr(void) { struct gicv5_irs_chip_data *irs_data; @@ -651,6 +657,18 @@ static int __init gicv5_irs_of_init_affinity(struct device_node *node, return ret; } +static void __init gicv5_irs_clear_affinity(struct gicv5_irs_chip_data *irs_data) +{ + int cpu; + + for_each_possible_cpu(cpu) { + if (per_cpu(per_cpu_irs_data, cpu) == irs_data) { + per_cpu(cpu_iaffid, cpu).valid = false; + per_cpu(per_cpu_irs_data, cpu) = NULL; + } + } +} + static void irs_setup_pri_bits(u32 idr1) { switch (FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1)) { @@ -722,6 +740,7 @@ static int __init gicv5_irs_init(struct gicv5_irs_chip_data *irs_data) static int __init gicv5_irs_of_init(struct device_node *node) { struct gicv5_irs_chip_data *irs_data; + struct resource res; void __iomem *irs_base; u8 iaffid_bits; u32 idr; @@ -739,6 +758,11 @@ static int __init gicv5_irs_of_init(struct device_node *node) goto out_err; } + if (of_address_to_resource(node, ret, &res)) { + ret = -EINVAL; + goto out_err; + } + irs_base = of_io_request_and_map(node, ret, of_node_full_name(node)); if (IS_ERR(irs_base)) { pr_err("%pOF: unable to map GICv5 IRS registers\n", node); @@ -747,6 +771,7 @@ static int __init gicv5_irs_of_init(struct device_node *node) } irs_data->fwnode = of_fwnode_handle(node); + irs_data->res = res; gicv5_irs_init_bases(irs_data, irs_base, of_property_read_bool(node, "dma-noncoherent")); idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1); @@ -773,7 +798,10 @@ static int __init gicv5_irs_of_init(struct device_node *node) return ret; out_iomem: + gicv5_irs_clear_affinity(irs_data); + gicv5_irs_disable(irs_data); iounmap(irs_base); + release_mem_region(res.start, resource_size(&res)); out_err: kfree(irs_data); return ret; @@ -787,7 +815,10 @@ void __init gicv5_irs_remove(void) gicv5_deinit_lpis(); list_for_each_entry_safe(irs_data, tmp_data, &irs_nodes, entry) { + gicv5_irs_clear_affinity(irs_data); + gicv5_irs_disable(irs_data); iounmap(irs_data->irs_base); + release_mem_region(irs_data->res.start, resource_size(&irs_data->res)); list_del(&irs_data->entry); kfree(irs_data); } @@ -860,6 +891,8 @@ static int __init gic_acpi_parse_iaffid(union acpi_subtable_headers *header, return 0; cpu = get_logical_index(gicc->arm_mpidr); + if (cpu < 0) + return 0; if (gicc->iaffid & ~GENMASK(current_iaffid_bits - 1, 0)) { pr_warn("CPU %d iaffid 0x%x exceeds IRS iaffid bits\n", cpu, gicc->iaffid); @@ -934,6 +967,7 @@ static int __init gic_acpi_parse_madt_irs(union acpi_subtable_headers *header, } gicv5_irs_init_bases(irs_data, irs_base, irs->flags & ACPI_MADT_IRS_NON_COHERENT); + irs_data->res = *r; gicv5_irs_acpi_init_affinity(irs->irs_id, irs_data); @@ -951,6 +985,8 @@ static int __init gic_acpi_parse_madt_irs(union acpi_subtable_headers *header, return 0; out_map: + gicv5_irs_clear_affinity(irs_data); + gicv5_irs_disable(irs_data); iounmap(irs_base); out_release: release_mem_region(r->start, resource_size(r)); diff --git a/drivers/irqchip/irq-gic-v5.c b/drivers/irqchip/irq-gic-v5.c index e9d1795235a66f..d6f1c0c8b74732 100644 --- a/drivers/irqchip/irq-gic-v5.c +++ b/drivers/irqchip/irq-gic-v5.c @@ -87,25 +87,13 @@ static void gicv5_ppi_priority_init(void) static void gicv5_hwirq_init(irq_hw_number_t hwirq, u8 priority, u8 hwirq_type) { - u64 cdpri, cdaff; - u16 iaffid; - int ret; + u64 cdpri; if (hwirq_type == GICV5_HWIRQ_TYPE_LPI || hwirq_type == GICV5_HWIRQ_TYPE_SPI) { cdpri = FIELD_PREP(GICV5_GIC_CDPRI_PRIORITY_MASK, priority) | FIELD_PREP(GICV5_GIC_CDPRI_TYPE_MASK, hwirq_type) | FIELD_PREP(GICV5_GIC_CDPRI_ID_MASK, hwirq); gic_insn(cdpri, CDPRI); - - ret = gicv5_irs_cpu_to_iaffid(smp_processor_id(), &iaffid); - - if (WARN_ON_ONCE(ret)) - return; - - cdaff = FIELD_PREP(GICV5_GIC_CDAFF_IAFFID_MASK, iaffid) | - FIELD_PREP(GICV5_GIC_CDAFF_TYPE_MASK, hwirq_type) | - FIELD_PREP(GICV5_GIC_CDAFF_ID_MASK, hwirq); - gic_insn(cdaff, CDAFF); } } @@ -548,6 +536,7 @@ static const struct irq_chip gicv5_spi_irq_chip = { .irq_get_irqchip_state = gicv5_spi_irq_get_irqchip_state, .irq_set_irqchip_state = gicv5_spi_irq_set_irqchip_state, .flags = IRQCHIP_SET_TYPE_MASKED | + IRQCHIP_AFFINITY_PRE_STARTUP | IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, }; @@ -561,7 +550,8 @@ static const struct irq_chip gicv5_lpi_irq_chip = { .irq_retrigger = gicv5_lpi_irq_retrigger, .irq_get_irqchip_state = gicv5_lpi_irq_get_irqchip_state, .irq_set_irqchip_state = gicv5_lpi_irq_set_irqchip_state, - .flags = IRQCHIP_SKIP_SET_WAKE | + .flags = IRQCHIP_AFFINITY_PRE_STARTUP | + IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, }; @@ -862,6 +852,9 @@ void __init gicv5_init_lpi_domain(void) void __init gicv5_free_lpi_domain(void) { + if (!gicv5_global_data.lpi_domain) + return; + irq_domain_remove(gicv5_global_data.lpi_domain); gicv5_global_data.lpi_domain = NULL; } @@ -983,6 +976,7 @@ static void gicv5_cpu_disable_interrupts(void) cr0 = FIELD_PREP(ICC_CR0_EL1_EN, 0); write_sysreg_s(cr0, SYS_ICC_CR0_EL1); + isb(); } static void gicv5_cpu_enable_interrupts(void) @@ -1170,7 +1164,7 @@ static int __init gicv5_init_common(struct fwnode_handle *parent_domain) ret = gicv5_starting_cpu(smp_processor_id()); if (ret) - goto out_dom; + goto out_int; ret = set_handle_irq(gicv5_handle_irq); if (ret) @@ -1178,16 +1172,17 @@ static int __init gicv5_init_common(struct fwnode_handle *parent_domain) ret = gicv5_irs_enable(); if (ret) - goto out_int; + goto out_handle; gicv5_smp_init(); gicv5_irs_its_probe(); return 0; +out_handle: + set_handle_irq(NULL); out_int: gicv5_cpu_disable_interrupts(); -out_dom: gicv5_free_domains(); return ret; } diff --git a/drivers/irqchip/irq-realtek-rtl.c b/drivers/irqchip/irq-realtek-rtl.c index 2ae3be7fa6338f..90fb1a14c1f136 100644 --- a/drivers/irqchip/irq-realtek-rtl.c +++ b/drivers/irqchip/irq-realtek-rtl.c @@ -1,5 +1,10 @@ // SPDX-License-Identifier: GPL-2.0-only /* + * Realtek Interrupt controller. + * + * The Realtek Interrupt controller is a big endian device found in the + * Realtek MIPS SoCs. + * * Copyright (C) 2020 Birger Koblitz * Copyright (C) 2020 Bert Vermeulen * Copyright (C) 2020 John Crispin @@ -25,6 +30,11 @@ #define REG(cpu, x) (realtek_ictl_base[cpu] + x) +struct realtek_ictl_output { + struct irq_domain *domain; + u32 mask; +}; + static DEFINE_RAW_SPINLOCK(irq_lock); static void __iomem *realtek_ictl_base[NR_CPUS]; @@ -41,18 +51,18 @@ static inline void enable_gimr(unsigned int cpu, unsigned int hw_irq) { u32 gimr; - gimr = readl(REG(cpu, RTL_ICTL_GIMR)); + gimr = readl_be(REG(cpu, RTL_ICTL_GIMR)); gimr |= BIT(hw_irq); - writel(gimr, REG(cpu, RTL_ICTL_GIMR)); + writel_be(gimr, REG(cpu, RTL_ICTL_GIMR)); } static inline void disable_gimr(unsigned int cpu, unsigned int hw_irq) { u32 gimr; - gimr = readl(REG(cpu, RTL_ICTL_GIMR)); + gimr = readl_be(REG(cpu, RTL_ICTL_GIMR)); gimr &= ~BIT(hw_irq); - writel(gimr, REG(cpu, RTL_ICTL_GIMR)); + writel_be(gimr, REG(cpu, RTL_ICTL_GIMR)); } static void write_irr(unsigned int cpu, int hw_irq, u32 value) @@ -62,9 +72,9 @@ static void write_irr(unsigned int cpu, int hw_irq, u32 value) unsigned int shift = IRR_SHIFT(hw_irq); u32 irr; - irr = readl(irr0 + offset) & ~(0xf << shift); + irr = readl_be(irr0 + offset) & ~(0xf << shift); irr |= (value & 0xf) << shift; - writel(irr, irr0 + offset); + writel_be(irr, irr0 + offset); } static void realtek_ictl_unmask_irq(struct irq_data *i) @@ -105,15 +115,17 @@ static struct irq_chip realtek_ictl_irq = { .irq_set_affinity = realtek_ictl_irq_affinity, }; -static int intc_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw) +static int intc_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw_irq) { + struct realtek_ictl_output *output = d->host_data; unsigned int cpu; irq_set_chip_and_handler(irq, &realtek_ictl_irq, handle_level_irq); guard(raw_spinlock_irqsave)(&irq_lock); + output->mask |= BIT(hw_irq); for_each_present_cpu(cpu) - write_irr(cpu, hw, 1); + write_irr(cpu, hw_irq, 1); return 0; } @@ -125,76 +137,99 @@ static const struct irq_domain_ops irq_domain_ops = { static void realtek_irq_dispatch(struct irq_desc *desc) { + struct realtek_ictl_output *output = irq_desc_get_handler_data(desc); struct irq_chip *chip = irq_desc_get_chip(desc); unsigned int cpu = smp_processor_id(); - struct irq_domain *domain; unsigned long pending; - unsigned int soc_int; + unsigned int hw_irq; chained_irq_enter(chip, desc); - pending = readl(REG(cpu, RTL_ICTL_GIMR)) & readl(REG(cpu, RTL_ICTL_GISR)); + pending = readl_be(REG(cpu, RTL_ICTL_GIMR)) & + readl_be(REG(cpu, RTL_ICTL_GISR)) & output->mask; if (unlikely(!pending)) { spurious_interrupt(); goto out; } - domain = irq_desc_get_handler_data(desc); - for_each_set_bit(soc_int, &pending, RTL_ICTL_NUM_INPUTS) - generic_handle_domain_irq(domain, soc_int); + for_each_set_bit(hw_irq, &pending, RTL_ICTL_NUM_INPUTS) + generic_handle_domain_irq(output->domain, hw_irq); out: chained_irq_exit(chip, desc); } -static int __init realtek_rtl_of_init(struct device_node *node, struct device_node *parent) +static int __init realtek_setup_parents(struct device_node *node) { + int err, parent_irq, num_parents = of_irq_count(node); + struct realtek_ictl_output *output; struct of_phandle_args oirq; struct irq_domain *domain; - int cpu, parent_irq; - for_each_present_cpu(cpu) { - realtek_ictl_base[cpu] = of_iomap(node, cpu); - if (!realtek_ictl_base[cpu]) - return -ENXIO; - - /* Disable all cascaded interrupts and clear routing */ - for (unsigned int hw_irq = 0; hw_irq < RTL_ICTL_NUM_INPUTS; hw_irq++) { - disable_gimr(cpu, hw_irq); - write_irr(cpu, hw_irq, 0); - } - } + output = kcalloc(1, sizeof(*output), GFP_KERNEL); + if (!output) + return -ENOMEM; - if (WARN_ON(!of_irq_count(node))) { + if (WARN_ON(!num_parents)) { /* - * If DT contains no parent interrupts, assume MIPS CPU IRQ 2 - * (HW0) is connected to the first output. This is the case for - * all known hardware anyway. "interrupt-map" is deprecated, so - * don't bother trying to parse that. + * If DT contains no parent interrupts, assume MIPS IRQ 2 (HW0) is + * connected to the first output. This is the case for all known hardware. */ - oirq.np = of_find_compatible_node(NULL, NULL, "mti,cpu-interrupt-controller"); + oirq.np = of_find_compatible_node(NULL, NULL, + "mti,cpu-interrupt-controller"); + if (!oirq.np) { + err = -EINVAL; + goto err_out; + } + oirq.args_count = 1; oirq.args[0] = 2; - parent_irq = irq_create_of_mapping(&oirq); - of_node_put(oirq.np); } else { parent_irq = of_irq_get(node, 0); } - if (parent_irq < 0) - return parent_irq; - else if (!parent_irq) - return -ENODEV; + if (parent_irq <= 0) { + err = parent_irq ? parent_irq : -ENODEV; + goto err_out; + } - domain = irq_domain_create_linear(of_fwnode_handle(node), RTL_ICTL_NUM_INPUTS, &irq_domain_ops, NULL); - if (!domain) - return -ENOMEM; + domain = irq_domain_create_linear(of_fwnode_handle(node), RTL_ICTL_NUM_INPUTS, + &irq_domain_ops, output); + if (!domain) { + err = -ENOMEM; + goto err_out; + } - irq_set_chained_handler_and_data(parent_irq, realtek_irq_dispatch, domain); + output->domain = domain; + irq_set_chained_handler_and_data(parent_irq, realtek_irq_dispatch, output); return 0; + +err_out: + kfree(output); + + return err; +} + +static int __init realtek_rtl_of_init(struct device_node *node, struct device_node *parent) +{ + unsigned int cpu; + + for_each_present_cpu(cpu) { + realtek_ictl_base[cpu] = of_iomap(node, cpu); + if (!realtek_ictl_base[cpu]) + return -ENXIO; + + /* Disable all cascaded interrupts and clear routing */ + for (unsigned int hw_irq = 0; hw_irq < RTL_ICTL_NUM_INPUTS; hw_irq++) { + disable_gimr(cpu, hw_irq); + write_irr(cpu, hw_irq, 0); + } + } + + return realtek_setup_parents(node); } IRQCHIP_DECLARE(realtek_rtl_intc, "realtek,rtl-intc", realtek_rtl_of_init); diff --git a/drivers/irqchip/irq-renesas-irqc.c b/drivers/irqchip/irq-renesas-irqc.c index a20a6471b0e48b..1ff3535a4617fa 100644 --- a/drivers/irqchip/irq-renesas-irqc.c +++ b/drivers/irqchip/irq-renesas-irqc.c @@ -176,6 +176,7 @@ static int irqc_probe(struct platform_device *pdev) goto err_runtime_pm_disable; } + p->irq_domain->flags |= IRQ_DOMAIN_FLAG_DESTROY_GC; ret = irq_alloc_domain_generic_chips(p->irq_domain, p->number_of_irqs, 1, "irqc", handle_level_irq, 0, 0, IRQ_GC_INIT_NESTED_LOCK); diff --git a/drivers/irqchip/irq-renesas-rzg2l.c b/drivers/irqchip/irq-renesas-rzg2l.c index f6b2e69a2f4e00..b3774f82855abd 100644 --- a/drivers/irqchip/irq-renesas-rzg2l.c +++ b/drivers/irqchip/irq-renesas-rzg2l.c @@ -161,7 +161,7 @@ static void rzg2l_clear_irq_int(struct rzg2l_irqc_priv *priv, unsigned int hwirq * falling/rising-edge. */ if ((iscr & bit) && (iitsr & IITSR_IITSEL_MASK(hw_irq))) { - writel_relaxed(iscr & ~bit, priv->base + ISCR); + writel_relaxed(~bit, priv->base + ISCR); /* * Enforce that the posted write is flushed to prevent that the * just handled interrupt is raised again. @@ -177,7 +177,7 @@ static void rzg2l_clear_tint_int(struct rzg2l_irqc_priv *priv, unsigned int hwir reg = readl_relaxed(priv->base + TSCR); if (reg & bit) { - writel_relaxed(reg & ~bit, priv->base + TSCR); + writel_relaxed(~bit, priv->base + TSCR); /* * Enforce that the posted write is flushed to prevent that the * just handled interrupt is raised again. diff --git a/drivers/leds/leds-gpio.c b/drivers/leds/leds-gpio.c index 8ae71c2e91e012..8810fdcf2d774f 100644 --- a/drivers/leds/leds-gpio.c +++ b/drivers/leds/leds-gpio.c @@ -277,6 +277,7 @@ static int gpio_led_probe(struct platform_device *pdev) if (IS_ERR(led_dat->gpiod)) { dev_info(dev, "Skipping unavailable LED gpio %d (%s)\n", template->gpio, template->name); + led_dat->gpiod = NULL; continue; } diff --git a/drivers/leds/leds-pca9532.c b/drivers/leds/leds-pca9532.c index f3bf59495b68d1..b2e081c8f13938 100644 --- a/drivers/leds/leds-pca9532.c +++ b/drivers/leds/leds-pca9532.c @@ -327,9 +327,9 @@ static int pca9532_gpio_set_value(struct gpio_chip *gc, unsigned int offset, struct pca9532_led *led = &data->leds[offset]; if (val) - led->state = PCA9532_ON; - else led->state = PCA9532_OFF; + else + led->state = PCA9532_ON; pca9532_setled(led); @@ -349,7 +349,7 @@ static int pca9532_gpio_get_value(struct gpio_chip *gc, unsigned offset) static int pca9532_gpio_direction_input(struct gpio_chip *gc, unsigned offset) { /* To use as input ensure pin is not driven */ - pca9532_gpio_set_value(gc, offset, 0); + pca9532_gpio_set_value(gc, offset, 1); return 0; } @@ -397,10 +397,14 @@ static int pca9532_configure(struct i2c_client *client, for (i = 0; i < 2; i++) { data->pwm[i] = pdata->pwm[i]; data->psc[i] = pdata->psc[i]; - i2c_smbus_write_byte_data(client, PCA9532_REG_PWM(maxleds, i), - data->pwm[i]); - i2c_smbus_write_byte_data(client, PCA9532_REG_PSC(maxleds, i), - data->psc[i]); + err = i2c_smbus_write_byte_data(client, PCA9532_REG_PWM(maxleds, i), + data->pwm[i]); + if (err < 0) + return err; + err = i2c_smbus_write_byte_data(client, PCA9532_REG_PSC(maxleds, i), + data->psc[i]); + if (err < 0) + return err; } data->hw_blink = true; diff --git a/drivers/leds/leds-st1202.c b/drivers/leds/leds-st1202.c index 7f68d956f6948c..2593ff39f22a6a 100644 --- a/drivers/leds/leds-st1202.c +++ b/drivers/leds/leds-st1202.c @@ -31,10 +31,11 @@ #define ST1202_ILED_REG0 0x09 #define ST1202_MAX_LEDS 12 #define ST1202_MAX_PATTERNS 8 -#define ST1202_MILLIS_PATTERN_DUR_MAX 5660 +#define ST1202_MILLIS_PATTERN_DUR_MAX (ST1202_MILLIS_PATTERN_DUR_MIN * U8_MAX) #define ST1202_MILLIS_PATTERN_DUR_MIN 22 #define ST1202_PATTERN_DUR 0x16 #define ST1202_PATTERN_PWM 0x1E +#define ST1202_PATTERN_PWM_FULL 0x0FFF #define ST1202_PATTERN_REP 0x15 struct st1202_led { @@ -85,7 +86,7 @@ static int st1202_write_reg(struct st1202_chip *chip, int reg, uint8_t val) static uint8_t st1202_prescalar_to_miliseconds(unsigned int value) { - return value / ST1202_MILLIS_PATTERN_DUR_MIN - 1; + return value / ST1202_MILLIS_PATTERN_DUR_MIN; } static int st1202_pwm_pattern_write(struct st1202_chip *chip, int led_num, @@ -127,37 +128,11 @@ static int st1202_duration_pattern_write(struct st1202_chip *chip, int pattern, st1202_prescalar_to_miliseconds(value)); } -static void st1202_brightness_set(struct led_classdev *led_cdev, - enum led_brightness value) -{ - struct st1202_led *led = cdev_to_st1202_led(led_cdev); - struct st1202_chip *chip = led->chip; - - guard(mutex)(&chip->lock); - - st1202_write_reg(chip, ST1202_ILED_REG0 + led->led_num, value); -} - -static enum led_brightness st1202_brightness_get(struct led_classdev *led_cdev) -{ - struct st1202_led *led = cdev_to_st1202_led(led_cdev); - struct st1202_chip *chip = led->chip; - u8 value = 0; - - guard(mutex)(&chip->lock); - - st1202_read_reg(chip, ST1202_ILED_REG0 + led->led_num, &value); - - return value; -} - -static int st1202_channel_set(struct st1202_chip *chip, int led_num, bool active) +static int __st1202_channel_set(struct st1202_chip *chip, int led_num, bool active) { u8 chan_low, chan_high; int ret; - guard(mutex)(&chip->lock); - if (led_num <= 7) { ret = st1202_read_reg(chip, ST1202_CHAN_ENABLE_LOW, &chan_low); if (ret < 0) @@ -185,6 +160,40 @@ static int st1202_channel_set(struct st1202_chip *chip, int led_num, bool active return 0; } +static int st1202_channel_set(struct st1202_chip *chip, int led_num, bool active) +{ + guard(mutex)(&chip->lock); + + return __st1202_channel_set(chip, led_num, active); +} + +static void st1202_brightness_set(struct led_classdev *led_cdev, + enum led_brightness value) +{ + struct st1202_led *led = cdev_to_st1202_led(led_cdev); + struct st1202_chip *chip = led->chip; + + guard(mutex)(&chip->lock); + + for (int patt = 0; patt < ST1202_MAX_PATTERNS; patt++) + st1202_pwm_pattern_write(chip, led->led_num, patt, ST1202_PATTERN_PWM_FULL); + st1202_write_reg(chip, ST1202_ILED_REG0 + led->led_num, value); + __st1202_channel_set(chip, led->led_num, !!value); +} + +static enum led_brightness st1202_brightness_get(struct led_classdev *led_cdev) +{ + struct st1202_led *led = cdev_to_st1202_led(led_cdev); + struct st1202_chip *chip = led->chip; + u8 value = 0; + + guard(mutex)(&chip->lock); + + st1202_read_reg(chip, ST1202_ILED_REG0 + led->led_num, &value); + + return value; +} + static int st1202_led_set(struct led_classdev *ldev, enum led_brightness value) { struct st1202_led *led = cdev_to_st1202_led(ldev); @@ -200,12 +209,16 @@ static int st1202_led_pattern_clear(struct led_classdev *ldev) guard(mutex)(&chip->lock); + ret = st1202_write_reg(chip, ST1202_CONFIG_REG, ST1202_CONFIG_REG_SHFT); + if (ret != 0) + return ret; + for (int patt = 0; patt < ST1202_MAX_PATTERNS; patt++) { - ret = st1202_pwm_pattern_write(chip, led->led_num, patt, LED_OFF); + ret = st1202_pwm_pattern_write(chip, led->led_num, patt, ST1202_PATTERN_PWM_FULL); if (ret != 0) return ret; - ret = st1202_duration_pattern_write(chip, patt, ST1202_MILLIS_PATTERN_DUR_MIN); + ret = st1202_write_reg(chip, ST1202_PATTERN_DUR + patt, 0); if (ret != 0) return ret; } @@ -226,6 +239,10 @@ static int st1202_led_pattern_set(struct led_classdev *ldev, guard(mutex)(&chip->lock); + ret = st1202_write_reg(chip, ST1202_CONFIG_REG, ST1202_CONFIG_REG_SHFT); + if (ret != 0) + return ret; + for (int patt = 0; patt < len; patt++) { if (pattern[patt].delta_t < ST1202_MILLIS_PATTERN_DUR_MIN || pattern[patt].delta_t > ST1202_MILLIS_PATTERN_DUR_MAX) @@ -244,6 +261,10 @@ static int st1202_led_pattern_set(struct led_classdev *ldev, if (ret != 0) return ret; + ret = __st1202_channel_set(chip, led->led_num, true); + if (ret != 0) + return ret; + ret = st1202_write_reg(chip, ST1202_CONFIG_REG, (ST1202_CONFIG_REG_PATSR | ST1202_CONFIG_REG_PATS | ST1202_CONFIG_REG_SHFT)); if (ret != 0) @@ -256,13 +277,19 @@ static int st1202_dt_init(struct st1202_chip *chip) { struct device *dev = &chip->client->dev; struct st1202_led *led; - int err, reg; + int err; + u32 reg; for_each_available_child_of_node_scoped(dev_of_node(dev), child) { err = of_property_read_u32(child, "reg", ®); if (err) return dev_err_probe(dev, err, "Invalid register\n"); + if (reg >= ST1202_MAX_LEDS) + return dev_err_probe(dev, -EINVAL, + "LED reg %u out of range [0, %d]\n", + reg, ST1202_MAX_LEDS - 1); + led = &chip->leds[reg]; led->is_active = true; led->fwnode = of_fwnode_handle(child); @@ -322,11 +349,6 @@ static int st1202_setup(struct st1202_chip *chip) if (ret < 0) return ret; - ret = st1202_write_reg(chip, ST1202_CONFIG_REG, - ST1202_CONFIG_REG_PATS | ST1202_CONFIG_REG_PATSR); - if (ret < 0) - return ret; - return 0; } diff --git a/drivers/leds/rgb/leds-lp5860-core.c b/drivers/leds/rgb/leds-lp5860-core.c index fd0e2f6e6e0f4b..e21d5f2302be0c 100644 --- a/drivers/leds/rgb/leds-lp5860-core.c +++ b/drivers/leds/rgb/leds-lp5860-core.c @@ -204,9 +204,9 @@ int lp5860_device_init(struct device *dev) mutex_lock(&lp->lock); ret = regmap_update_bits(lp->regmap, LP5860_REG_DEV_INITIAL, LP5860_MODE_MASK, LP5860_MODE_1 << LP5860_MODE_SHIFT); + mutex_unlock(&lp->lock); if (ret) goto err_disable; - mutex_unlock(&lp->lock); ret = lp5860_init_dt(lp); if (ret) @@ -215,7 +215,6 @@ int lp5860_device_init(struct device *dev) return 0; err_disable: - mutex_unlock(&lp->lock); lp5860_chip_enable(lp, LP5860_CHIP_DISABLE); return ret; } diff --git a/drivers/leds/rgb/leds-lp5860-spi.c b/drivers/leds/rgb/leds-lp5860-spi.c index 5e0c44854a680d..6bf6a625c28af7 100644 --- a/drivers/leds/rgb/leds-lp5860-spi.c +++ b/drivers/leds/rgb/leds-lp5860-spi.c @@ -38,6 +38,7 @@ static int lp5860_probe(struct spi_device *spi) struct device *dev = &spi->dev; struct lp5860 *lp5860; unsigned int multi_leds; + int ret; multi_leds = device_get_child_node_count(dev); if (!multi_leds) { @@ -61,7 +62,10 @@ static int lp5860_probe(struct spi_device *spi) "Failed to initialise Regmap.\n"); lp5860->dev = dev; - mutex_init(&lp5860->lock); + + ret = devm_mutex_init(dev, &lp5860->lock); + if (ret) + return ret; spi_set_drvdata(spi, lp5860); @@ -70,10 +74,6 @@ static int lp5860_probe(struct spi_device *spi) static void lp5860_remove(struct spi_device *spi) { - struct lp5860 *lp5860 = spi_get_drvdata(spi); - - mutex_destroy(&lp5860->lock); - lp5860_device_remove(&spi->dev); } diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c index 636879ae1db76b..7a8e749374b5cc 100644 --- a/drivers/mailbox/pcc.c +++ b/drivers/mailbox/pcc.c @@ -91,12 +91,11 @@ struct pcc_chan_reg { * @plat_irq: platform interrupt * @type: PCC subspace type * @plat_irq_flags: platform interrupt flags - * @chan_in_use: this flag is used just to check if the interrupt needs - * handling when it is shared. Since only one transfer can occur - * at a time and mailbox takes care of locking, this flag can be - * accessed without a lock. Note: the type only support the - * communication from OSPM to Platform, like type3, use it, and - * other types completely ignore it. + * @chan_in_use: lockless flag used by type 3 initiator subspaces to filter + * platform interrupts. Only one transfer can occur at a time, but + * the interrupt handler may sample the flag on another CPU, so all + * accesses must use READ_ONCE() or WRITE_ONCE(). Other subspace + * types do not test it. */ struct pcc_chan_info { struct pcc_mbox_chan chan; @@ -320,8 +319,13 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) if (pcc_chan_reg_read_modify_write(&pchan->plat_irq_ack)) return IRQ_NONE; + /* + * Initiator subspaces use this flag to filter shared interrupts. Use + * READ_ONCE() to sample the lockless flag written by pcc_send_data() + * on another CPU. + */ if (pchan->type == ACPI_PCCT_TYPE_EXT_PCC_MASTER_SUBSPACE && - !pchan->chan_in_use) + !READ_ONCE(pchan->chan_in_use)) return IRQ_NONE; if (!pcc_mbox_cmd_complete_check(pchan)) @@ -331,12 +335,12 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) return IRQ_NONE; /* - * Clear this flag after updating interrupt ack register and just - * before mbox_chan_received_data() which might call pcc_send_data() - * where the flag is set again to start new transfer. This is - * required to avoid any possible race in updatation of this flag. + * Clear this flag after updating the interrupt ack register and before + * notifying the client and mailbox core. mbox_chan_txdone() may submit + * the next queued transfer and set the flag again. Use WRITE_ONCE() for + * the lockless update observed by the send and interrupt paths. */ - pchan->chan_in_use = false; + WRITE_ONCE(pchan->chan_in_use, false); mbox_chan_received_data(chan, NULL); mbox_chan_txdone(chan, 0); @@ -438,9 +442,18 @@ static int pcc_send_data(struct mbox_chan *chan, void *data) if (ret) return ret; + /* + * Set chan_in_use before ringing the doorbell so a fast completion + * interrupt is not mistaken for a shared interrupt from another + * subspace. Use WRITE_ONCE() for the lockless flag update. The + * ordered I/O accessor used to ring the doorbell orders this store + * before the platform is notified. + */ + if (pchan->plat_irq > 0) + WRITE_ONCE(pchan->chan_in_use, true); ret = pcc_chan_reg_read_modify_write(&pchan->db); - if (!ret && pchan->plat_irq > 0) - pchan->chan_in_use = true; + if (ret && pchan->plat_irq > 0) + WRITE_ONCE(pchan->chan_in_use, false); return ret; } diff --git a/drivers/mailbox/qcom-cpucp-mbox.c b/drivers/mailbox/qcom-cpucp-mbox.c index 862e45e8fbd58b..298b357c0f9a60 100644 --- a/drivers/mailbox/qcom-cpucp-mbox.c +++ b/drivers/mailbox/qcom-cpucp-mbox.c @@ -63,14 +63,25 @@ static irqreturn_t qcom_cpucp_mbox_irq_fn(int irq, void *data) for_each_set_bit(i, (unsigned long *)&status, cpucp->mbox.num_chans) { u32 val = readl(cpucp->rx_base + APSS_CPUCP_RX_MBOX_CMD(i) + APSS_CPUCP_MBOX_CMD_OFF); struct mbox_chan *chan = &cpucp->chans[i]; + struct mbox_client *cl; unsigned long flags; - /* Provide mutual exclusion with changes to chan->cl */ + /* + * Provide mutual exclusion with changes to chan->cl. + * Save cl locally and clear the HW interrupt inside the lock, + * then invoke mbox_chan_received_data() outside the lock to + * avoid a PREEMPT_RT self-deadlock: mbox_chan_received_data() + * can call back into mbox_send_message() via scmi_rx_callback() + * -> mailbox_clear_channel(), which re-acquires chan->lock + * (converted to an rt_spinlock under PREEMPT_RT). + */ spin_lock_irqsave(&chan->lock, flags); - if (chan->cl) - mbox_chan_received_data(chan, &val); + cl = chan->cl; writeq(BIT(i), cpucp->rx_base + APSS_CPUCP_RX_MBOX_CLEAR); spin_unlock_irqrestore(&chan->lock, flags); + + if (cl) + mbox_chan_received_data(chan, &val); } return IRQ_HANDLED; @@ -106,6 +117,14 @@ static int qcom_cpucp_mbox_send_data(struct mbox_chan *chan, void *data) unsigned long chan_id = channel_number(chan); u32 *val = data; + /* + * mailbox_clear_channel() calls mbox_send_message() with NULL data to + * signal the remote side that the channel has been cleared. Nothing + * needs to be written to the TX register in that case, so just return. + */ + if (!val) + return 0; + writel(*val, cpucp->tx_base + APSS_CPUCP_TX_MBOX_CMD(chan_id) + APSS_CPUCP_MBOX_CMD_OFF); return 0; diff --git a/drivers/mailbox/riscv-sbi-mpxy-mbox.c b/drivers/mailbox/riscv-sbi-mpxy-mbox.c index 7c9c006b7244a5..714f7fb97a2fd6 100644 --- a/drivers/mailbox/riscv-sbi-mpxy-mbox.c +++ b/drivers/mailbox/riscv-sbi-mpxy-mbox.c @@ -314,8 +314,13 @@ static int mpxy_get_notifications(u32 channel_id, channel_id, 0, 0, 0, 0, 0); if (sret.error) goto err_put_cpu; + if (sret.value < 0 || mpxy_shmem_size < sizeof(*notif_data) || + sret.value > mpxy_shmem_size - sizeof(*notif_data)) { + put_cpu(); + return -EOVERFLOW; + } - memcpy(notif_data, mpxy->shmem, sret.value + 16); + memcpy(notif_data, mpxy->shmem, sret.value + sizeof(*notif_data)); *events_data_len = sret.value; err_put_cpu: @@ -480,11 +485,14 @@ static void mpxy_mbox_peek_rpmi_data(struct mbox_chan *chan, struct rpmi_mbox_message msg; unsigned long pos = 0; - while (pos < events_data_len && (events_data_len - pos) <= sizeof(*event)) { + while (events_data_len - pos >= sizeof(*event)) { event = (struct rpmi_notification_event *)(notif->events_data + pos); msg.type = RPMI_MBOX_MSG_TYPE_NOTIFICATION_EVENT; msg.notif.event_datalen = le16_to_cpu(event->event_datalen); + if (msg.notif.event_datalen > + events_data_len - pos - sizeof(*event)) + break; msg.notif.event_id = event->event_id; msg.notif.event_data = event->event_data; msg.error = 0; diff --git a/drivers/mailbox/rockchip-mailbox.c b/drivers/mailbox/rockchip-mailbox.c index a1a7dee643567c..d55cbf7f2cb6d8 100644 --- a/drivers/mailbox/rockchip-mailbox.c +++ b/drivers/mailbox/rockchip-mailbox.c @@ -196,19 +196,10 @@ static int rockchip_mbox_probe(struct platform_device *pdev) /* Each channel has two buffers for A2B and B2A */ mb->buf_size = (size_t)resource_size(res) / (drv_data->num_chans * 2); - mb->pclk = devm_clk_get(&pdev->dev, "pclk_mailbox"); - if (IS_ERR(mb->pclk)) { - ret = PTR_ERR(mb->pclk); - dev_err(&pdev->dev, "failed to get pclk_mailbox clock: %d\n", - ret); - return ret; - } - - ret = clk_prepare_enable(mb->pclk); - if (ret) { - dev_err(&pdev->dev, "failed to enable pclk: %d\n", ret); - return ret; - } + mb->pclk = devm_clk_get_enabled(&pdev->dev, "pclk_mailbox"); + if (IS_ERR(mb->pclk)) + return dev_err_probe(&pdev->dev, PTR_ERR(mb->pclk), + "failed to get and enable pclk_mailbox clock\n"); for (i = 0; i < mb->mbox.num_chans; i++) { irq = platform_get_irq(pdev, i); diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 81d3f42c4f48c7..86fc35e76a0f7a 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -66,6 +66,7 @@ #define SB_VERSION_4 4 #define SB_VERSION_5 5 #define SB_VERSION_6 6 +#define SB_VERSION_7 7 #define SB_SECTORS 8 #define MAX_SECTORS_PER_BLOCK 8 @@ -91,6 +92,7 @@ struct superblock { #define SB_FLAG_FIXED_PADDING 0x8 #define SB_FLAG_FIXED_HMAC 0x10 #define SB_FLAG_INLINE 0x20 +#define SB_FLAG_DISCARD_KEYED 0x40 #define JOURNAL_ENTRY_ROUNDUP 8 @@ -277,6 +279,7 @@ struct dm_integrity_c { bool recalculate_flag; bool reset_recalculate_flag; bool discard; + bool discard_keyed; bool fix_padding; bool fix_hmac; bool legacy_recalculate; @@ -483,7 +486,9 @@ static void wraparound_section(struct dm_integrity_c *ic, unsigned int *sec_ptr) static void sb_set_version(struct dm_integrity_c *ic) { - if (ic->sb->flags & cpu_to_le32(SB_FLAG_INLINE)) + if (ic->sb->flags & cpu_to_le32(SB_FLAG_DISCARD_KEYED)) + ic->sb->version = SB_VERSION_7; + else if (ic->sb->flags & cpu_to_le32(SB_FLAG_INLINE)) ic->sb->version = SB_VERSION_6; else if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) ic->sb->version = SB_VERSION_5; @@ -1416,7 +1421,7 @@ static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, se { unsigned int hash_offset = 0; unsigned char mismatch_hash = 0; - unsigned char mismatch_filler = !ic->discard; + unsigned char mismatch_filler = !ic->discard || ic->discard_keyed; do { unsigned char *data, *dp; @@ -1468,7 +1473,7 @@ thorough_test: } hash_offset = 0; mismatch_hash = 0; - mismatch_filler = !ic->discard; + mismatch_filler = !ic->discard || ic->discard_keyed; } } } @@ -1646,7 +1651,8 @@ static void integrity_end_io(struct bio *bio) } static void integrity_sector_checksum_shash(struct dm_integrity_c *ic, sector_t sector, - const char *data, unsigned offset, char *result) + const char *data, unsigned offset, + unsigned int len, char *result) { __le64 sector_le = cpu_to_le64(sector); SHASH_DESC_ON_STACK(req, ic->internal_shash); @@ -1675,10 +1681,12 @@ static void integrity_sector_checksum_shash(struct dm_integrity_c *ic, sector_t goto failed; } - r = crypto_shash_update(req, data + offset, ic->sectors_per_block << SECTOR_SHIFT); - if (unlikely(r < 0)) { - dm_integrity_io_error(ic, "crypto_shash_update", r); - goto failed; + if (likely(len)) { + r = crypto_shash_update(req, data + offset, len); + if (unlikely(r < 0)) { + dm_integrity_io_error(ic, "crypto_shash_update", r); + goto failed; + } } r = crypto_shash_final(req, result); @@ -1699,7 +1707,8 @@ failed: } static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ahash_request **ahash_req, - sector_t sector, struct page *page, unsigned offset, char *result) + sector_t sector, struct page *page, unsigned offset, + unsigned int len, char *result) { __le64 sector_le = cpu_to_le64(sector); struct ahash_request *req; @@ -1708,6 +1717,7 @@ static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ah int r; unsigned int digest_size; unsigned int nbytes = 0; + unsigned int nents = 1 + (len ? 1 : 0); might_sleep(); @@ -1721,12 +1731,12 @@ static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ah ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, crypto_req_done, &wait); if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) { - sg_init_table(sg, 3); + sg_init_table(sg, nents + 1); sg_set_buf(s, (const __u8 *)&ic->sb->salt, SALT_SIZE); nbytes += SALT_SIZE; s++; } else { - sg_init_table(sg, 2); + sg_init_table(sg, nents); } if (likely(!is_vmalloc_addr(§or_le))) { @@ -1739,8 +1749,10 @@ static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ah nbytes += sizeof(sector_le); s++; - sg_set_page(s, page, ic->sectors_per_block << SECTOR_SHIFT, offset); - nbytes += ic->sectors_per_block << SECTOR_SHIFT; + if (likely(len)) { + sg_set_page(s, page, len, offset); + nbytes += len; + } ahash_request_set_crypt(req, sg, result, nbytes); @@ -1763,11 +1775,41 @@ failed: static void integrity_sector_checksum(struct dm_integrity_c *ic, struct ahash_request **ahash_req, sector_t sector, const char *data, unsigned offset, char *result) +{ + unsigned int len = ic->sectors_per_block << SECTOR_SHIFT; + + if (likely(ic->internal_shash != NULL)) + integrity_sector_checksum_shash(ic, sector, data, offset, len, result); + else + integrity_sector_checksum_ahash(ic, ahash_req, sector, (struct page *)data, + offset, len, result); +} + +/* + * Authenticated marker for a discarded block: HMAC_key(salt || sector), with + * no data payload. Because a real data tag's input always covers a full + * block, its length differs from this marker's, so the two can never + * collide structurally, regardless of block content. + */ +static void integrity_discard_checksum(struct dm_integrity_c *ic, struct ahash_request **ahash_req, + sector_t sector, char *result) { if (likely(ic->internal_shash != NULL)) - integrity_sector_checksum_shash(ic, sector, data, offset, result); + integrity_sector_checksum_shash(ic, sector, NULL, 0, 0, result); else - integrity_sector_checksum_ahash(ic, ahash_req, sector, (struct page *)data, offset, result); + integrity_sector_checksum_ahash(ic, ahash_req, sector, NULL, 0, 0, result); +} + +static void integrity_discard_fill_tags(struct dm_integrity_c *ic, struct ahash_request **ahash_req, + unsigned char *checksums, sector_t *sector, + unsigned int blocks) +{ + unsigned int i; + + for (i = 0; i < blocks; i++) { + integrity_discard_checksum(ic, ahash_req, *sector, checksums + i * ic->tag_size); + *sector += ic->sectors_per_block; + } } static void *integrity_kmap(struct dm_integrity_c *ic, struct page *p) @@ -1796,6 +1838,29 @@ static void *integrity_identity(struct dm_integrity_c *ic, void *data) return virt_to_page(data); } +static int integrity_recheck_verify_tag(struct dm_integrity_io *dio, char *checksum, + char *on_disk_tag, sector_t logical_sector) +{ + struct dm_integrity_c *ic = dio->ic; + int r; + + if (!ic->discard_keyed) + return dm_integrity_rw_tag(ic, checksum, &dio->metadata_block, + &dio->metadata_offset, ic->tag_size, TAG_CMP); + + r = dm_integrity_rw_tag(ic, on_disk_tag, &dio->metadata_block, + &dio->metadata_offset, ic->tag_size, TAG_READ); + if (unlikely(r)) + return r; + + r = crypto_memneq(on_disk_tag, checksum, ic->tag_size); + if (unlikely(r)) { + integrity_discard_checksum(ic, &dio->ahash_req, logical_sector, checksum); + r = crypto_memneq(on_disk_tag, checksum, ic->tag_size); + } + return r; +} + static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checksum) { struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); @@ -1821,6 +1886,7 @@ static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checks char *mem; char *buffer = page_to_virt(page); unsigned int buffer_offset; + char on_disk_tag[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; int r; struct dm_io_request io_req; struct dm_io_region io_loc; @@ -1848,8 +1914,8 @@ static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checks } integrity_sector_checksum(ic, &dio->ahash_req, logical_sector, integrity_identity(ic, buffer), buffer_offset, checksum); - r = dm_integrity_rw_tag(ic, checksum, &dio->metadata_block, - &dio->metadata_offset, ic->tag_size, TAG_CMP); + r = integrity_recheck_verify_tag(dio, checksum, on_disk_tag, + logical_sector); if (r) { if (r > 0) { DMERR_LIMIT("%pg: Checksum failed at sector 0x%llx", @@ -1914,14 +1980,19 @@ static void integrity_metadata(struct work_struct *w) if (unlikely(dio->op == REQ_OP_DISCARD)) { unsigned int bi_size = dio->bio_details.bi_iter.bi_size; unsigned int max_size = likely(checksums != checksums_onstack) ? PAGE_SIZE : HASH_MAX_DIGESTSIZE; - unsigned int max_blocks = max_size / ic->tag_size; + unsigned int max_blocks = (max_size - extra_space) / ic->tag_size; + sector_t sector = dio->range.logical_sector; - memset(checksums, DISCARD_FILLER, max_size); + if (!ic->discard_keyed) + memset(checksums, DISCARD_FILLER, max_size); while (bi_size) { unsigned int this_step_blocks = bi_size >> (SECTOR_SHIFT + ic->sb->log2_sectors_per_block); this_step_blocks = min(this_step_blocks, max_blocks); + if (ic->discard_keyed) + integrity_discard_fill_tags(ic, &dio->ahash_req, checksums, + §or, this_step_blocks); r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset, this_step_blocks * ic->tag_size, TAG_WRITE); if (unlikely(r)) { @@ -3798,6 +3869,8 @@ static void dm_integrity_resume(struct dm_target *ti) ic->wrote_to_journal = false; flags = ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING); + if (ic->discard_keyed) + flags |= cpu_to_le32(SB_FLAG_DISCARD_KEYED); r = sync_rw_sb(ic, REQ_OP_READ); if (r) dm_integrity_io_error(ic, "reading superblock", r); @@ -3945,7 +4018,8 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type, arg_count += ic->sectors_per_block != 1; arg_count += !!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)); arg_count += ic->reset_recalculate_flag; - arg_count += ic->discard; + arg_count += ic->discard && !ic->discard_keyed; + arg_count += ic->discard_keyed; arg_count += ic->mode != 'I'; /* interleave_sectors */ arg_count += ic->mode == 'J'; /* journal_sectors */ arg_count += ic->mode == 'J'; /* journal_watermark */ @@ -3968,8 +4042,10 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type, DMEMIT(" recalculate"); if (ic->reset_recalculate_flag) DMEMIT(" reset_recalculate"); - if (ic->discard) + if (ic->discard && !ic->discard_keyed) DMEMIT(" allow_discards"); + if (ic->discard_keyed) + DMEMIT(" allow_discards_keyed"); if (ic->mode != 'I') DMEMIT(" interleave_sectors:%u", 1U << ic->sb->log2_interleave_sectors); DMEMIT(" buffer_sectors:%u", 1U << ic->log2_buffer_sectors); @@ -4019,6 +4095,7 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type, DMEMIT(",recalculate=%c", (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) ? 'y' : 'n'); DMEMIT(",allow_discards=%c", ic->discard ? 'y' : 'n'); + DMEMIT(",allow_discards_keyed=%c", ic->discard_keyed ? 'y' : 'n'); DMEMIT(",fix_padding=%c", ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0) ? 'y' : 'n'); DMEMIT(",fix_hmac=%c", @@ -4176,6 +4253,9 @@ static int initialize_superblock(struct dm_integrity_c *ic, get_random_bytes(ic->sb->salt, SALT_SIZE); } + if (ic->discard_keyed) + ic->sb->flags |= cpu_to_le32(SB_FLAG_DISCARD_KEYED); + if (!ic->meta_dev) { if (ic->fix_padding) ic->sb->flags |= cpu_to_le32(SB_FLAG_FIXED_PADDING); @@ -4833,6 +4913,9 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv ic->reset_recalculate_flag = true; } else if (!strcmp(opt_string, "allow_discards")) { ic->discard = true; + } else if (!strcmp(opt_string, "allow_discards_keyed")) { + ic->discard = true; + ic->discard_keyed = true; } else if (!strcmp(opt_string, "fix_padding")) { ic->fix_padding = true; } else if (!strcmp(opt_string, "fix_hmac")) { @@ -4961,6 +5044,11 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv ti->error = "Discard can be only used with internal hash"; goto bad; } + if (ic->discard_keyed && !ic->internal_hash_alg.key) { + r = -EINVAL; + ti->error = "Keyed discard can only be used with keyed internal hash"; + goto bad; + } ic->autocommit_jiffies = msecs_to_jiffies(sync_msec); ic->autocommit_msec = sync_msec; @@ -5079,7 +5167,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv should_write_sb = true; } - if (!ic->sb->version || ic->sb->version > SB_VERSION_6) { + if (!ic->sb->version || ic->sb->version > SB_VERSION_7) { r = -EINVAL; ti->error = "Unknown version"; goto bad; @@ -5127,6 +5215,11 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv goto bad; } } + if (!ic->discard_keyed && (ic->sb->flags & cpu_to_le32(SB_FLAG_DISCARD_KEYED))) { + r = -EINVAL; + ti->error = "Keyed discard cannot be disabled once enabled"; + goto bad; + } if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) != !!ic->journal_mac_alg.alg_string) { r = -EINVAL; ti->error = "Journal mac mismatch"; @@ -5442,7 +5535,7 @@ static void dm_integrity_dtr(struct dm_target *ti) static struct target_type integrity_target = { .name = "integrity", - .version = {1, 14, 0}, + .version = {1, 15, 0}, .module = THIS_MODULE, .features = DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY, .ctr = dm_integrity_ctr, diff --git a/drivers/md/dm-pcache/cache.c b/drivers/md/dm-pcache/cache.c index e68dacb69459c3..b0b3e21677deeb 100644 --- a/drivers/md/dm-pcache/cache.c +++ b/drivers/md/dm-pcache/cache.c @@ -118,6 +118,9 @@ int cache_pos_decode(struct pcache_cache *cache, if (!latest_addr) return -EIO; + if (!cache_seg_id_valid(cache, latest.cache_seg_id)) + return -EIO; + pos->cache_seg = &cache->segments[latest.cache_seg_id]; if (latest.seg_off >= pos->cache_seg->segment.data_size) @@ -159,6 +162,7 @@ static int cache_init(struct dm_pcache *pcache) cache->cache_dev = &pcache->cache_dev; cache->n_segs = cache_dev->seg_num; atomic_set(&cache->gc_errors, 0); + atomic_set(&cache->writeback_errors, 0); spin_lock_init(&cache->seg_map_lock); spin_lock_init(&cache->key_head_lock); @@ -198,6 +202,7 @@ static int cache_tail_init(struct pcache_cache *cache) { struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); bool new_cache = !(cache->cache_info.flags & PCACHE_CACHE_FLAGS_INIT_DONE); + int ret; if (new_cache) { __set_bit(0, cache->seg_map); @@ -214,6 +219,12 @@ static int cache_tail_init(struct pcache_cache *cache) pcache_dev_err(pcache, "Corrupted key tail or dirty tail.\n"); return -EIO; } + + ret = cache_verify_dirty_tail(cache); + if (ret) { + pcache_dev_err(pcache, "dirty tail chain does not terminate (crafted cache image?)\n"); + return ret; + } } return 0; diff --git a/drivers/md/dm-pcache/cache.h b/drivers/md/dm-pcache/cache.h index afc112b79496ac..8809ec5ae943c3 100644 --- a/drivers/md/dm-pcache/cache.h +++ b/drivers/md/dm-pcache/cache.h @@ -180,6 +180,7 @@ struct pcache_cache { u32 advance; int ret; } writeback_ctx; + atomic_t writeback_errors; char gc_kset_onmedia_buf[PCACHE_KSET_ONMEDIA_SIZE_MAX]; struct delayed_work gc_work; @@ -420,6 +421,20 @@ static inline bool cache_seg_is_ctrl_seg(u32 cache_seg_id) return (cache_seg_id == 0); } +/** + * cache_seg_id_valid - Validate a cache segment id read from the cache device. + * @cache: Pointer to the pcache_cache structure. + * @cache_seg_id: Segment id decoded from on-media metadata. + * + * On-media segment ids are only protected by a CRC, which an attacker who can + * format the cache device computes over their chosen value. Reject any id that + * would index cache->segments[] out of bounds before it is dereferenced. + */ +static inline bool cache_seg_id_valid(struct pcache_cache *cache, u32 cache_seg_id) +{ + return cache_seg_id < cache->cache_info.n_segs; +} + /** * cache_key_cutfront - Cuts a specified length from the front of a cache key. * @key: Pointer to pcache_cache_key structure. @@ -651,6 +666,8 @@ static inline int cache_decode_dirty_tail(struct pcache_cache *cache) &cache->dirty_tail_index); } +int cache_verify_dirty_tail(struct pcache_cache *cache); + int pcache_cache_init(void); void pcache_cache_exit(void); #endif /* _PCACHE_CACHE_H */ diff --git a/drivers/md/dm-pcache/cache_gc.c b/drivers/md/dm-pcache/cache_gc.c index 3088cfef951bc1..a167892be7cef6 100644 --- a/drivers/md/dm-pcache/cache_gc.c +++ b/drivers/md/dm-pcache/cache_gc.c @@ -74,11 +74,17 @@ static bool need_gc(struct pcache_cache *cache, struct pcache_cache_pos *dirty_t * @cache: Pointer to the pcache_cache structure. * @kset_onmedia: Pointer to the kset_onmedia structure for the last kset. */ -static void last_kset_gc(struct pcache_cache *cache, struct pcache_cache_kset_onmedia *kset_onmedia) +static int last_kset_gc(struct pcache_cache *cache, struct pcache_cache_kset_onmedia *kset_onmedia) { struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); struct pcache_cache_segment *cur_seg, *next_seg; + if (!cache_seg_id_valid(cache, kset_onmedia->next_cache_seg_id)) { + pcache_dev_err(pcache, "invalid next_cache_seg_id %u in gc (n_segs %u)\n", + kset_onmedia->next_cache_seg_id, cache->n_segs); + return -EIO; + } + cur_seg = cache->key_tail.cache_seg; next_seg = &cache->segments[kset_onmedia->next_cache_seg_id]; @@ -94,6 +100,8 @@ static void last_kset_gc(struct pcache_cache *cache, struct pcache_cache_kset_on spin_lock(&cache->seg_map_lock); __clear_bit(cur_seg->cache_seg_id, cache->seg_map); spin_unlock(&cache->seg_map_lock); + + return 0; } void pcache_cache_gc_fn(struct work_struct *work) @@ -130,7 +138,11 @@ void pcache_cache_gc_fn(struct work_struct *work) if (dirty_tail.cache_seg == key_tail.cache_seg) break; - last_kset_gc(cache, kset_onmedia); + ret = last_kset_gc(cache, kset_onmedia); + if (ret) { + atomic_inc(&cache->gc_errors); + return; + } continue; } diff --git a/drivers/md/dm-pcache/cache_key.c b/drivers/md/dm-pcache/cache_key.c index 51dd1ccf4e2fde..a0d109f85504d3 100644 --- a/drivers/md/dm-pcache/cache_key.c +++ b/drivers/md/dm-pcache/cache_key.c @@ -94,6 +94,12 @@ int cache_key_decode(struct pcache_cache *cache, key->off = key_onmedia->off; key->len = key_onmedia->len; + if (!cache_seg_id_valid(cache, key_onmedia->cache_seg_id)) { + pcache_dev_err(pcache, "invalid cache_seg_id %u in cache key (n_segs %u)\n", + key_onmedia->cache_seg_id, cache->n_segs); + return -EIO; + } + key->cache_pos.cache_seg = &cache->segments[key_onmedia->cache_seg_id]; key->cache_pos.seg_off = key_onmedia->cache_seg_off; @@ -801,6 +807,11 @@ int cache_replay(struct pcache_cache *cache) goto out; } + if (!cache_seg_id_valid(cache, kset_onmedia->next_cache_seg_id)) { + ret = -EIO; + goto out; + } + next_seg = &cache->segments[kset_onmedia->next_cache_seg_id]; pos->cache_seg = next_seg; @@ -832,6 +843,75 @@ out: return ret; } +/* + * cache_verify_dirty_tail - reject a persisted dirty_tail whose last-kset + * chain does not terminate. + * + * dirty_tail is decoded independently of the key_tail chain cache_replay() + * walks, so replay's hop cap does not cover it. A crafted chain that loops + * back on itself makes the writeback worker re-arm forever; walk it once here + * with the same cap and fail the load if it does not end within n_segs hops. + */ +int cache_verify_dirty_tail(struct pcache_cache *cache) +{ + struct pcache_cache_pos pos; + struct pcache_cache_kset_onmedia *kset_onmedia; + u32 to_copy, last_hops = 0, count = 0; + int ret = 0; + + kset_onmedia = kzalloc(PCACHE_KSET_ONMEDIA_SIZE_MAX, GFP_KERNEL); + if (!kset_onmedia) + return -ENOMEM; + + cache_pos_copy(&pos, &cache->dirty_tail); + + while (true) { + to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, cache_seg_remain(&pos)); + ret = copy_mc_to_kernel(kset_onmedia, cache_pos_addr(&pos), to_copy); + if (ret) { + ret = -EIO; + goto out; + } + + /* A missing, short or corrupt kset is the normal end of the chain. */ + if (!kset_onmedia_valid(kset_onmedia) || + kset_onmedia->crc != cache_kset_crc(kset_onmedia)) { + ret = 0; + goto out; + } + + if (kset_onmedia->flags & PCACHE_KSET_FLAGS_LAST) { + if (kset_onmedia->next_cache_seg_id >= cache->cache_info.n_segs) { + ret = -EIO; + goto out; + } + + if (++last_hops > cache->n_segs) { + ret = -EIO; + goto out; + } + + pos.cache_seg = &cache->segments[kset_onmedia->next_cache_seg_id]; + pos.seg_off = 0; + continue; + } + + if (get_kset_onmedia_size(kset_onmedia) > cache_seg_remain(&pos)) { + ret = -EIO; + goto out; + } + + cache_pos_advance(&pos, get_kset_onmedia_size(kset_onmedia)); + if (++count > 512) { + cond_resched(); + count = 0; + } + } +out: + kfree(kset_onmedia); + return ret; +} + int cache_tree_init(struct pcache_cache *cache, struct pcache_cache_tree *cache_tree, u32 n_subtrees) { int ret; diff --git a/drivers/md/dm-pcache/cache_writeback.c b/drivers/md/dm-pcache/cache_writeback.c index 3df8687e2382f7..c34db79fcefb43 100644 --- a/drivers/md/dm-pcache/cache_writeback.c +++ b/drivers/md/dm-pcache/cache_writeback.c @@ -196,12 +196,18 @@ clear_tree: return ret; } -static void last_kset_writeback(struct pcache_cache *cache, +static int last_kset_writeback(struct pcache_cache *cache, struct pcache_cache_kset_onmedia *last_kset_onmedia) { struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); struct pcache_cache_segment *next_seg; + if (!cache_seg_id_valid(cache, last_kset_onmedia->next_cache_seg_id)) { + pcache_dev_err(pcache, "invalid next_cache_seg_id %u in writeback (n_segs %u)\n", + last_kset_onmedia->next_cache_seg_id, cache->n_segs); + return -EIO; + } + pcache_dev_debug(pcache, "last kset, next: %u\n", last_kset_onmedia->next_cache_seg_id); next_seg = &cache->segments[last_kset_onmedia->next_cache_seg_id]; @@ -211,6 +217,8 @@ static void last_kset_writeback(struct pcache_cache *cache, cache->dirty_tail.seg_off = 0; cache_encode_dirty_tail(cache); mutex_unlock(&cache->dirty_tail_lock); + + return 0; } void cache_writeback_fn(struct work_struct *work) @@ -229,6 +237,9 @@ void cache_writeback_fn(struct work_struct *work) if (pcache_is_stopping(pcache)) goto unlock; + if (atomic_read(&cache->writeback_errors)) + goto unlock; + kset_onmedia = (struct pcache_cache_kset_onmedia *)cache->wb_kset_onmedia_buf; mutex_lock(&cache->dirty_tail_lock); @@ -241,15 +252,19 @@ void cache_writeback_fn(struct work_struct *work) } if (kset_onmedia->flags & PCACHE_KSET_FLAGS_LAST) { - last_kset_writeback(cache, kset_onmedia); + ret = last_kset_writeback(cache, kset_onmedia); + if (ret) { + atomic_inc(&cache->writeback_errors); + goto unlock; + } delay = 0; goto queue_work; } ret = cache_kset_insert_tree(cache, kset_onmedia); if (ret) { - delay = PCACHE_CACHE_WRITEBACK_INTERVAL; - goto queue_work; + atomic_inc(&cache->writeback_errors); + goto unlock; } cache_wb_tree_writeback(cache, get_kset_onmedia_size(kset_onmedia)); diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 0f02e2956398da..7e4fbca93ccb2c 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -2064,23 +2064,18 @@ static void bitmap_end_behind_write(struct mddev *mddev) bitmap->mddev->bitmap_info.max_write_behind); } -static bool bitmap_wait_behind_writes(struct mddev *mddev, bool nowait) +static void bitmap_wait_behind_writes(struct mddev *mddev) { struct bitmap *bitmap = mddev->bitmap; /* wait for behind writes to complete */ if (bitmap && atomic_read(&bitmap->behind_writes) > 0) { - if (nowait) - return false; - pr_debug("md:%s: behind writes in progress - waiting to stop.\n", mdname(mddev)); /* need to kick something here to make sure I/O goes? */ wait_event(bitmap->behind_wait, atomic_read(&bitmap->behind_writes) == 0); } - - return true; } static void bitmap_destroy(struct mddev *mddev) @@ -2090,7 +2085,7 @@ static void bitmap_destroy(struct mddev *mddev) if (!bitmap) /* there was no bitmap */ return; - bitmap_wait_behind_writes(mddev, false); + bitmap_wait_behind_writes(mddev); if (!test_bit(MD_SERIALIZE_POLICY, &mddev->flags)) mddev_destroy_serial_pool(mddev, NULL); @@ -2629,10 +2624,12 @@ static ssize_t location_store(struct mddev *mddev, const char *buf, size_t len) { int rv; + unsigned int noio_flags; rv = mddev_suspend_and_lock(mddev); if (rv) return rv; + noio_flags = memalloc_noio_save(); if (mddev->pers) { if (mddev->recovery || mddev->sync_thread) { @@ -2719,6 +2716,7 @@ location_store(struct mddev *mddev, const char *buf, size_t len) } rv = 0; out: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); if (rv) return rv; @@ -2862,7 +2860,7 @@ backlog_store(struct mddev *mddev, const char *buf, size_t len) if (!has_write_mostly) { pr_warn_ratelimited("%s: can't set backlog, no write mostly device available\n", mdname(mddev)); - mddev_unlock(mddev); + mddev_unlock_and_resume(mddev); return -EINVAL; } diff --git a/drivers/md/md-bitmap.h b/drivers/md/md-bitmap.h index f46674bdfeb918..890276d9c66ef8 100644 --- a/drivers/md/md-bitmap.h +++ b/drivers/md/md-bitmap.h @@ -29,6 +29,7 @@ enum bitmap_state { BITMAP_FIRST_USE = 3, /* llbitmap is just created */ BITMAP_CLEAN = 4, /* llbitmap is created with assume_clean */ BITMAP_DAEMON_BUSY = 5, /* llbitmap daemon is not finished after daemon_sleep */ + BITMAP_SHUTDOWN = 6, /* llbitmap is being destroyed */ BITMAP_HOSTENDIAN =15, }; @@ -98,7 +99,7 @@ struct bitmap_operations { void (*start_behind_write)(struct mddev *mddev); void (*end_behind_write)(struct mddev *mddev); - bool (*wait_behind_writes)(struct mddev *mddev, bool nowait); + void (*wait_behind_writes)(struct mddev *mddev); md_bitmap_fn *start_write; md_bitmap_fn *end_write; diff --git a/drivers/md/md-linear.c b/drivers/md/md-linear.c index fdff250d0d5137..73b367b61b873b 100644 --- a/drivers/md/md-linear.c +++ b/drivers/md/md-linear.c @@ -71,6 +71,7 @@ static int linear_set_limits(struct mddev *mddev) int err; md_init_stacking_limits(&lim); + lim.features |= BLK_FEAT_NOWAIT; lim.max_hw_sectors = mddev->chunk_sectors; lim.logical_block_size = mddev->logical_block_size; lim.max_write_zeroes_sectors = mddev->chunk_sectors; diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 5a4e2abaa75770..e0bc6c8c945923 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -789,6 +789,7 @@ write_bitmap: if (state == BitNeedSync || state == BitNeedSyncUnwritten) need_resync = !mddev->degraded; else if (state == BitDirty && + !test_bit(BITMAP_SHUTDOWN, &llbitmap->flags) && !timer_pending(&llbitmap->pending_timer)) mod_timer(&llbitmap->pending_timer, jiffies + mddev->bitmap_info.daemon_sleep * HZ); @@ -981,7 +982,7 @@ static int llbitmap_read_sb(struct llbitmap *llbitmap) else mddev->bitmap_info.space = mddev->bitmap_info.default_space; } - llbitmap->flags = le32_to_cpu(sb->state); + llbitmap->flags = le32_to_cpu(sb->state) & ~BIT(BITMAP_SHUTDOWN); if (test_and_clear_bit(BITMAP_FIRST_USE, &llbitmap->flags)) { ret = llbitmap_init(llbitmap); goto out_put_page; @@ -1037,6 +1038,9 @@ static void llbitmap_pending_timer_fn(struct timer_list *pending_timer) struct llbitmap *llbitmap = container_of(pending_timer, struct llbitmap, pending_timer); + if (test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) + return; + if (work_busy(&llbitmap->daemon_work)) { pr_warn("md/llbitmap: %s daemon_work not finished in %lu seconds\n", mdname(llbitmap->mddev), @@ -1057,6 +1061,9 @@ static void md_llbitmap_daemon_fn(struct work_struct *work) bool restart; int idx; + if (test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) + return; + if (llbitmap->mddev->degraded) return; retry: @@ -1096,7 +1103,7 @@ retry: goto retry; /* If some page is dirty but not expired, setup timer again */ - if (restart) + if (restart && !test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) mod_timer(&llbitmap->pending_timer, jiffies + llbitmap->mddev->bitmap_info.daemon_sleep * HZ); } @@ -1126,10 +1133,11 @@ static int llbitmap_create(struct mddev *mddev) mutex_lock(&mddev->bitmap_info.mutex); mddev->bitmap = llbitmap; ret = llbitmap_read_sb(llbitmap); + if (ret) + mddev->bitmap = NULL; mutex_unlock(&mddev->bitmap_info.mutex); if (ret) { kfree(llbitmap); - mddev->bitmap = NULL; } return ret; @@ -1178,7 +1186,9 @@ static void llbitmap_destroy(struct mddev *mddev) mutex_lock(&mddev->bitmap_info.mutex); - timer_delete_sync(&llbitmap->pending_timer); + set_bit(BITMAP_SHUTDOWN, &llbitmap->flags); + timer_shutdown_sync(&llbitmap->pending_timer); + cancel_work_sync(&llbitmap->daemon_work); flush_workqueue(md_llbitmap_io_wq); flush_workqueue(md_llbitmap_unplug_wq); @@ -1517,7 +1527,7 @@ static void llbitmap_update_sb(void *data) sb = kmap_local_page(sb_page); sb->events = cpu_to_le64(mddev->events); - sb->state = cpu_to_le32(llbitmap->flags); + sb->state = cpu_to_le32(llbitmap->flags & ~BIT(BITMAP_SHUTDOWN)); sb->chunksize = cpu_to_le32(llbitmap->chunksize); sb->sync_size = cpu_to_le64(mddev->resync_max_sectors); sb->events_cleared = cpu_to_le64(llbitmap->events_cleared); @@ -1574,19 +1584,13 @@ static void llbitmap_end_behind_write(struct mddev *mddev) wake_up(&llbitmap->behind_wait); } -static bool llbitmap_wait_behind_writes(struct mddev *mddev, bool nowait) +static void llbitmap_wait_behind_writes(struct mddev *mddev) { struct llbitmap *llbitmap = mddev->bitmap; - if (llbitmap && atomic_read(&llbitmap->behind_writes) > 0) { - if (nowait) - return false; - + if (llbitmap && atomic_read(&llbitmap->behind_writes) > 0) wait_event(llbitmap->behind_wait, atomic_read(&llbitmap->behind_writes) == 0); - } - - return true; } static ssize_t bits_show(struct mddev *mddev, char *page) diff --git a/drivers/md/md.c b/drivers/md/md.c index 4dd133445539aa..b9aef7ba115154 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -233,23 +233,22 @@ static int rdev_need_serial(struct md_rdev *rdev) void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev) { int ret = 0; + unsigned int noio_flags; - if (rdev && !rdev_need_serial(rdev) && + if (!test_bit(MD_SERIALIZE_POLICY, &mddev->flags) && + rdev && !rdev_need_serial(rdev) && !test_bit(CollisionCheck, &rdev->flags)) return; + noio_flags = memalloc_noio_save(); if (!rdev) ret = rdevs_init_serial(mddev); else ret = rdev_init_serial(rdev); if (ret) - return; + goto out; if (mddev->serial_info_pool == NULL) { - /* - * already in memalloc noio context by - * mddev_suspend() - */ mddev->serial_info_pool = mempool_create_kmalloc_pool(NR_SERIAL_INFOS, sizeof(struct serial_info)); @@ -258,6 +257,8 @@ void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev) pr_err("can't alloc memory pool for serialization\n"); } } +out: + memalloc_noio_restore(noio_flags); } /* @@ -516,9 +517,6 @@ int mddev_suspend(struct mddev *mddev, bool interruptible) */ WRITE_ONCE(mddev->suspended, mddev->suspended + 1); - /* restrict memory reclaim I/O during raid array is suspend */ - mddev->noio_flag = memalloc_noio_save(); - mutex_unlock(&mddev->suspend_mutex); return 0; } @@ -535,9 +533,6 @@ static void __mddev_resume(struct mddev *mddev, bool recovery_needed) return; } - /* entred the memalloc scope from mddev_suspend() */ - memalloc_noio_restore(mddev->noio_flag); - percpu_ref_resurrect(&mddev->active_io); wake_up(&mddev->sb_wait); @@ -4054,6 +4049,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len) char clevel[16]; ssize_t rv; size_t slen = len; + unsigned int noio_flags; struct md_personality *pers, *oldpers; long level; void *priv, *oldpriv; @@ -4065,6 +4061,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len) rv = mddev_suspend_and_lock(mddev); if (rv) return rv; + noio_flags = memalloc_noio_save(); if (mddev->pers == NULL) { memcpy(mddev->clevel, buf, slen); @@ -4240,6 +4237,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len) md_new_event(); rv = len; out_unlock: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return rv; } @@ -4419,6 +4417,7 @@ static ssize_t raid_disks_store(struct mddev *mddev, const char *buf, size_t len) { unsigned int n; + unsigned int noio_flags; int err; err = kstrtouint(buf, 10, &n); @@ -4428,6 +4427,7 @@ raid_disks_store(struct mddev *mddev, const char *buf, size_t len) err = mddev_suspend_and_lock(mddev); if (err) return err; + noio_flags = memalloc_noio_save(); if (mddev->pers) { if (n != mddev->raid_disks) err = update_raid_disks(mddev, n); @@ -4451,6 +4451,7 @@ raid_disks_store(struct mddev *mddev, const char *buf, size_t len) } else mddev->raid_disks = n; out_unlock: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return err ? err : len; } @@ -4831,6 +4832,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) int minor; dev_t dev; struct md_rdev *rdev; + unsigned int noio_flags; int err; if (!*buf || *e != ':' || !e[1] || e[1] == '\n') @@ -4846,6 +4848,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) err = mddev_suspend_and_lock(mddev); if (err) return err; + noio_flags = memalloc_noio_save(); if (mddev->persistent) { rdev = md_import_device(dev, mddev->major_version, mddev->minor_version); @@ -4864,6 +4867,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) rdev = md_import_device(dev, -1, -1); if (IS_ERR(rdev)) { + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return PTR_ERR(rdev); } @@ -4871,6 +4875,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) out: if (err) export_rdev(rdev); + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); if (!err) md_new_event(); @@ -6290,7 +6295,7 @@ void md_init_stacking_limits(struct queue_limits *lim) { blk_set_stacking_limits(lim); lim->features = BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | - BLK_FEAT_IO_STAT | BLK_FEAT_NOWAIT; + BLK_FEAT_IO_STAT; } EXPORT_SYMBOL_GPL(md_init_stacking_limits); @@ -6638,7 +6643,6 @@ int md_run(struct mddev *mddev) int err; struct md_rdev *rdev; struct md_personality *pers; - bool nowait = true; if (list_empty(&mddev->disks)) /* cannot run an array with no devices.. */ @@ -6709,7 +6713,6 @@ int md_run(struct mddev *mddev) } } sysfs_notify_dirent_safe(rdev->sysfs_state); - nowait = nowait && bdev_nowait(rdev->bdev); } pers = get_pers(mddev->level, mddev->clevel); @@ -7057,7 +7060,7 @@ EXPORT_SYMBOL_GPL(md_stop_writes); static void mddev_detach(struct mddev *mddev) { if (md_bitmap_enabled(mddev, false)) - mddev->bitmap_ops->wait_behind_writes(mddev, false); + mddev->bitmap_ops->wait_behind_writes(mddev); if (mddev->pers && mddev->pers->quiesce && !is_md_suspended(mddev)) { mddev->pers->quiesce(mddev, 1); mddev->pers->quiesce(mddev, 0); @@ -7073,8 +7076,8 @@ static void __md_stop(struct mddev *mddev) { struct md_personality *pers = mddev->pers; - md_bitmap_destroy(mddev); mddev_detach(mddev); + md_bitmap_destroy(mddev); spin_lock(&mddev->lock); mddev->pers = NULL; spin_unlock(&mddev->lock); @@ -8331,8 +8334,10 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode, unsigned int cmd, unsigned long arg) { int err = 0; + unsigned int noio_flags = 0; void __user *argp = (void __user *)arg; struct mddev *mddev = NULL; + bool suspend; err = md_ioctl_valid(cmd); if (err) @@ -8382,13 +8387,15 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode, if (!md_is_rdwr(mddev)) flush_work(&mddev->sync_work); - err = md_ioctl_need_suspend(cmd) ? mddev_suspend_and_lock(mddev) : - mddev_lock(mddev); + suspend = md_ioctl_need_suspend(cmd); + err = suspend ? mddev_suspend_and_lock(mddev) : mddev_lock(mddev); if (err) { pr_debug("md: ioctl lock interrupted, reason %d, cmd %d\n", err, cmd); goto out; } + if (suspend) + noio_flags = memalloc_noio_save(); if (cmd == SET_ARRAY_INFO) { err = __md_set_array_info(mddev, argp); @@ -8513,8 +8520,12 @@ unlock: err != -EINVAL) mddev->hold_active = 0; - md_ioctl_need_suspend(cmd) ? mddev_unlock_and_resume(mddev) : - mddev_unlock(mddev); + if (suspend) { + memalloc_noio_restore(noio_flags); + mddev_unlock_and_resume(mddev); + } else { + mddev_unlock(mddev); + } out: if (cmd == STOP_ARRAY_RO || (err && cmd == STOP_ARRAY)) @@ -9428,6 +9439,8 @@ static void md_clone_bio(struct mddev *mddev, struct bio **bio) md_io_clone->mddev = mddev; if (blk_queue_io_stat(bdev->bd_disk->queue)) md_io_clone->start_time = bio_start_io_acct(*bio); + else + md_io_clone->start_time = 0; if (bio_data_dir(*bio) == WRITE && md_bitmap_enabled(mddev, false)) { md_io_clone->offset = (*bio)->bi_iter.bi_sector; @@ -10182,19 +10195,34 @@ static void md_start_sync(struct work_struct *ws) struct mddev *mddev = container_of(ws, struct mddev, sync_work); int spares = 0; bool suspend = false; + unsigned int noio_flags = 0; char *name; /* * If reshape is still in progress, spares won't be added or removed * from conf until reshape is done. */ - if (mddev->reshape_position == MaxSector && + if ((mddev->reshape_position == MaxSector || !md_is_rdwr(mddev)) && md_spares_need_change(mddev)) { suspend = true; mddev_suspend(mddev, false); + noio_flags = memalloc_noio_save(); } mddev_lock_nointr(mddev); + + /* + * The spare configuration can change before reconfig_mutex is acquired. + * Recheck while holding the lock and suspend if needed. + */ + if (!suspend && (mddev->reshape_position == MaxSector || !md_is_rdwr(mddev)) && + md_spares_need_change(mddev)) { + mddev_unlock(mddev); + mddev_suspend_and_lock_nointr(mddev); + suspend = true; + noio_flags = memalloc_noio_save(); + } + if (!md_is_rdwr(mddev)) { /* * On a read-only array we can: @@ -10238,8 +10266,10 @@ static void md_start_sync(struct work_struct *ws) * https://bugzilla.kernel.org/show_bug.cgi?id=218200 * Therefore, use __mddev_resume(mddev, false). */ - if (suspend) + if (suspend) { + memalloc_noio_restore(noio_flags); __mddev_resume(mddev, false); + } md_wakeup_thread(mddev->sync_thread); sysfs_notify_dirent_safe(mddev->sysfs_action); md_new_event(); @@ -10258,8 +10288,10 @@ not_running: * https://bugzilla.kernel.org/show_bug.cgi?id=218200 * Therefore, use __mddev_resume(mddev, false). */ - if (suspend) + if (suspend) { + memalloc_noio_restore(noio_flags); __mddev_resume(mddev, false); + } wake_up(&resync_wait); if (test_and_clear_bit(MD_RECOVERY_RECOVER, &mddev->recovery) && diff --git a/drivers/md/md.h b/drivers/md/md.h index d8daf0f75cbbe8..76488cd9e81ee8 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -621,7 +621,6 @@ struct mddev { struct md_cluster_info *cluster_info; struct md_cluster_operations *cluster_ops; unsigned int good_device_nr; /* good device num within cluster raid */ - unsigned int noio_flag; /* for memalloc scope API */ /* * Temporarily store rdev that will be finally removed when diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c index 2c000b3a5f4910..35e103f0c2c3e2 100644 --- a/drivers/md/raid0.c +++ b/drivers/md/raid0.c @@ -385,6 +385,7 @@ static int raid0_set_limits(struct mddev *mddev) int err; md_init_stacking_limits(&lim); + lim.features |= BLK_FEAT_NOWAIT; lim.max_hw_sectors = mddev->chunk_sectors; lim.max_write_zeroes_sectors = mddev->chunk_sectors; lim.max_hw_wzeroes_unmap_sectors = mddev->chunk_sectors; diff --git a/drivers/md/raid1-10.c b/drivers/md/raid1-10.c index 56a56a4da4f83f..3b0e230692ba96 100644 --- a/drivers/md/raid1-10.c +++ b/drivers/md/raid1-10.c @@ -290,9 +290,8 @@ static inline bool raid1_should_read_first(struct mddev *mddev, } /* - * bio with REQ_RAHEAD or REQ_NOWAIT can fail at anytime, before such IO is - * submitted to the underlying disks, hence don't record badblocks or retry - * in this case. + * bio with REQ_RAHEAD can fail at anytime, before such IO is submitted to the + * underlying disks, hence don't record badblocks or retry in this case. * * BLK_STS_INVAL means the bio was not valid for the underlying device. This * is a user error, not a device failure, so retrying or recording bad blocks @@ -300,6 +299,5 @@ static inline bool raid1_should_read_first(struct mddev *mddev, */ static inline bool raid1_should_handle_error(struct bio *bio) { - return !(bio->bi_opf & (REQ_RAHEAD | REQ_NOWAIT)) && - bio->bi_status != BLK_STS_INVAL; + return !(bio->bi_opf & REQ_RAHEAD) && bio->bi_status != BLK_STS_INVAL; } diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index afe2ca96ad8c22..1c50f363efc08f 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -1051,10 +1051,8 @@ static void lower_barrier(struct r1conf *conf, sector_t sector_nr) wake_up(&conf->wait_barrier); } -static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait) +static void _wait_barrier(struct r1conf *conf, int idx) { - bool ret = true; - /* * We need to increase conf->nr_pending[idx] very early here, * then raise_barrier() can be blocked when it waits for @@ -1085,7 +1083,7 @@ static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait) */ if (!READ_ONCE(conf->array_frozen) && !atomic_read(&conf->barrier[idx])) - return ret; + return; /* * After holding conf->resync_lock, conf->nr_pending[idx] @@ -1104,26 +1102,18 @@ static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait) wake_up_barrier(conf); /* Wait for the barrier in same barrier unit bucket to drop. */ - /* Return false when nowait flag is set */ - if (nowait) { - ret = false; - } else { - wait_event_lock_irq(conf->wait_barrier, - !conf->array_frozen && - !atomic_read(&conf->barrier[idx]), - conf->resync_lock); - atomic_inc(&conf->nr_pending[idx]); - } + wait_event_lock_irq(conf->wait_barrier, !conf->array_frozen && + !atomic_read(&conf->barrier[idx]), + conf->resync_lock); + atomic_inc(&conf->nr_pending[idx]); atomic_dec(&conf->nr_waiting[idx]); spin_unlock_irq(&conf->resync_lock); - return ret; } -static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait) +static void wait_read_barrier(struct r1conf *conf, sector_t sector_nr) { int idx = sector_to_idx(sector_nr); - bool ret = true; /* * Very similar to _wait_barrier(). The difference is, for read @@ -1135,7 +1125,7 @@ static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowa atomic_inc(&conf->nr_pending[idx]); if (!READ_ONCE(conf->array_frozen)) - return ret; + return; spin_lock_irq(&conf->resync_lock); atomic_inc(&conf->nr_waiting[idx]); @@ -1147,27 +1137,19 @@ static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowa wake_up_barrier(conf); /* Wait for array to be unfrozen */ - /* Return false when nowait flag is set */ - if (nowait) { - /* Return false when nowait flag is set */ - ret = false; - } else { - wait_event_lock_irq(conf->wait_barrier, - !conf->array_frozen, - conf->resync_lock); - atomic_inc(&conf->nr_pending[idx]); - } + wait_event_lock_irq(conf->wait_barrier, !conf->array_frozen, + conf->resync_lock); + atomic_inc(&conf->nr_pending[idx]); atomic_dec(&conf->nr_waiting[idx]); spin_unlock_irq(&conf->resync_lock); - return ret; } -static bool wait_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait) +static void wait_barrier(struct r1conf *conf, sector_t sector_nr) { int idx = sector_to_idx(sector_nr); - return _wait_barrier(conf, idx, nowait); + _wait_barrier(conf, idx); } static void _allow_barrier(struct r1conf *conf, int idx) @@ -1342,7 +1324,6 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, int max_sectors; int rdisk; bool r1bio_existed = !!r1_bio; - bool nowait = bio->bi_opf & REQ_NOWAIT; /* * An md cloned bio indicates we are in the error path. @@ -1362,16 +1343,7 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, * Still need barrier for READ in case that whole * array is frozen. */ - if (!wait_read_barrier(conf, bio->bi_iter.bi_sector, nowait)) { - bio_wouldblock_error(bio); - - if (r1bio_existed) { - set_bit(R1BIO_Returned, &r1_bio->state); - raid_end_bio_io(r1_bio); - } - - return; - } + wait_read_barrier(conf, bio->bi_iter.bi_sector); if (!r1_bio) r1_bio = alloc_r1bio(mddev, bio); @@ -1406,14 +1378,10 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, md_bitmap_enabled(mddev, false)) { /* * Reading from a write-mostly device must take care not to - * over-take any writes that are 'behind' - */ - mddev_add_trace_msg(mddev, "raid1 wait behind writes"); - if (!mddev->bitmap_ops->wait_behind_writes(mddev, nowait)) { - bio_wouldblock_error(bio); - set_bit(R1BIO_Returned, &r1_bio->state); - goto err_handle; - } + * over-take any writes that are 'behind' + */ + mddev_add_trace_msg(mddev, "raid1 wait behind writes"); + mddev->bitmap_ops->wait_behind_writes(mddev); } if (max_sectors < bio_sectors(bio)) { @@ -1435,7 +1403,6 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, } read_bio = bio_alloc_clone(mirror->rdev->bdev, bio, gfp, &mddev->bio_set); - read_bio->bi_opf &= ~REQ_NOWAIT; r1_bio->bios[rdisk] = read_bio; read_bio->bi_iter.bi_sector = r1_bio->sector + @@ -1454,7 +1421,7 @@ err_handle: raid_end_bio_io(r1_bio); } -static bool wait_blocked_rdev(struct mddev *mddev, struct bio *bio) +static void wait_blocked_rdev(struct mddev *mddev, struct bio *bio) { struct r1conf *conf = mddev->private; int disks = conf->raid_disks * 2; @@ -1474,9 +1441,6 @@ retry: set_bit(BlockedBadBlocks, &rdev->flags); if (rdev_blocked(rdev)) { - if (bio->bi_opf & REQ_NOWAIT) - return false; - mddev_add_trace_msg(rdev->mddev, "raid1 wait rdev %d blocked", rdev->raid_disk); atomic_inc(&rdev->nr_pending); @@ -1484,8 +1448,6 @@ retry: goto retry; } } - - return true; } static void raid1_start_write_behind(struct mddev *mddev, struct r1bio *r1_bio, @@ -1521,18 +1483,13 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, unsigned long flags; int first_clone; bool write_behind = false; - bool nowait = bio->bi_opf & REQ_NOWAIT; + bool atomic = bio->bi_opf & REQ_ATOMIC; bool is_discard = op_is_discard(bio->bi_opf); sector_t sector = bio->bi_iter.bi_sector; if (mddev_is_clustered(mddev) && mddev->cluster_ops->area_resyncing(mddev, WRITE, sector, bio_end_sector(bio))) { - - if (nowait) { - bio_wouldblock_error(bio); - return false; - } wait_event_idle(conf->wait_barrier, !mddev->cluster_ops->area_resyncing(mddev, WRITE, sector, @@ -1544,15 +1501,9 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, * thread has put up a bar for new requests. * Continue immediately if no resync is active currently. */ - if (!wait_barrier(conf, sector, nowait)) { - bio_wouldblock_error(bio); - return false; - } + wait_barrier(conf, sector); - if (!wait_blocked_rdev(mddev, bio)) { - bio_wouldblock_error(bio); - goto err_allow_barrier; - } + wait_blocked_rdev(mddev, bio); r1_bio = alloc_r1bio(mddev, bio); r1_bio->sectors = max_sectors; @@ -1579,6 +1530,8 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, */ if (!is_discard && rdev && test_bit(WriteMostly, &rdev->flags)) write_behind = true; + if (atomic && max_sectors > BIO_MAX_VECS * (PAGE_SIZE >> 9)) + write_behind = false; r1_bio->bios[i] = NULL; if (!rdev || test_bit(Faulty, &rdev->flags)) @@ -1604,19 +1557,6 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, if (is_bad) { int good_sectors; - /* - * We cannot atomically write this, so just - * error in that case. It could be possible to - * atomically write other mirrors, but the - * complexity of supporting that is not worth - * the benefit. - */ - if (bio->bi_opf & REQ_ATOMIC) { - bio->bi_status = BLK_STS_NOTSUPP; - bio_endio(bio); - goto err_dec_pending; - } - good_sectors = first_bad - sector; if (good_sectors < max_sectors) max_sectors = good_sectors; @@ -1637,6 +1577,11 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, max_sectors = min_t(int, max_sectors, BIO_MAX_VECS * (PAGE_SIZE >> 9)); if (max_sectors < bio_sectors(bio)) { + if (atomic) { + bio_io_error(bio); + goto err_dec_pending; + } + bio = bio_submit_split_bioset(bio, max_sectors, &conf->bio_split); if (!bio) @@ -1681,7 +1626,6 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, wait_for_serialization(rdev, r1_bio); } - mbio->bi_opf &= ~REQ_NOWAIT; r1_bio->bios[i] = mbio; mbio->bi_iter.bi_sector = sector + rdev->data_offset; @@ -1720,8 +1664,6 @@ err_dec_pending: } free_r1bio(r1_bio); - -err_allow_barrier: allow_barrier(conf, sector); return false; @@ -1850,7 +1792,7 @@ static void close_sync(struct r1conf *conf) int idx; for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++) { - _wait_barrier(conf, idx, false); + _wait_barrier(conf, idx); _allow_barrier(conf, idx); } @@ -3228,6 +3170,7 @@ static int raid1_set_limits(struct mddev *mddev) md_init_stacking_limits(&lim); lim.max_write_zeroes_sectors = 0; lim.max_hw_wzeroes_unmap_sectors = 0; + lim.chunk_sectors = BARRIER_UNIT_SECTOR_SIZE; lim.logical_block_size = mddev->logical_block_size; lim.features |= BLK_FEAT_ATOMIC_WRITES; lim.features |= BLK_FEAT_PCI_P2PDMA; @@ -3485,8 +3428,6 @@ static void *raid1_takeover(struct mddev *mddev) mddev->new_chunk_sectors = 0; conf = setup_conf(mddev); if (!IS_ERR(conf)) { - /* Array must appear to be quiesced */ - conf->array_frozen = 1; mddev_clear_unsupported_flags(mddev, UNSUPPORTED_MDDEV_FLAGS); } diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 54cddb3a98cd62..8c419033a9bf32 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -1002,32 +1002,22 @@ static bool wait_barrier_nolock(struct r10conf *conf) return false; } -static bool wait_barrier(struct r10conf *conf, bool nowait) +static void wait_barrier(struct r10conf *conf) { - bool ret = true; - if (wait_barrier_nolock(conf)) - return true; + return; write_seqlock_irq(&conf->resync_lock); if (conf->barrier) { - /* Return false when nowait flag is set */ - if (nowait) { - ret = false; - } else { - conf->nr_waiting++; - mddev_add_trace_msg(conf->mddev, "raid10 wait barrier"); - wait_event_barrier(conf, stop_waiting_barrier(conf)); - conf->nr_waiting--; - } + conf->nr_waiting++; + mddev_add_trace_msg(conf->mddev, "raid10 wait barrier"); + wait_event_barrier(conf, stop_waiting_barrier(conf)); + conf->nr_waiting--; if (!conf->nr_waiting) wake_up(&conf->wait_barrier); } - /* Only increment nr_pending when we wait */ - if (ret) - atomic_inc(&conf->nr_pending); + atomic_inc(&conf->nr_pending); write_sequnlock_irq(&conf->resync_lock); - return ret; } static void allow_barrier(struct r10conf *conf) @@ -1119,30 +1109,22 @@ static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule) * currently. * 2. If IO spans the reshape position. Need to wait for reshape to pass. */ -static bool regular_request_wait(struct mddev *mddev, struct r10conf *conf, +static void regular_request_wait(struct mddev *mddev, struct r10conf *conf, struct bio *bio, sector_t sectors) { - /* Bail out if REQ_NOWAIT is set for the bio */ - if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) { - bio_wouldblock_error(bio); - return false; - } + wait_barrier(conf); + while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && bio->bi_iter.bi_sector < conf->reshape_progress && bio->bi_iter.bi_sector + sectors > conf->reshape_progress) { allow_barrier(conf); - if (bio->bi_opf & REQ_NOWAIT) { - bio_wouldblock_error(bio); - return false; - } mddev_add_trace_msg(conf->mddev, "raid10 wait reshape"); wait_event(conf->wait_barrier, conf->reshape_progress <= bio->bi_iter.bi_sector || conf->reshape_progress >= bio->bi_iter.bi_sector + sectors); - wait_barrier(conf, false); + wait_barrier(conf); } - return true; } static void raid10_read_request(struct mddev *mddev, struct bio *bio, @@ -1191,10 +1173,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, } } - if (!regular_request_wait(mddev, conf, bio, r10_bio->sectors)) { - free_r10bio(r10_bio); - return; - } + regular_request_wait(mddev, conf, bio, r10_bio->sectors); rdev = read_balance(conf, r10_bio, &max_sectors); if (!rdev) { @@ -1215,7 +1194,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, allow_barrier(conf); bio = bio_submit_split_bioset(bio, max_sectors, &conf->bio_split); - wait_barrier(conf, false); + wait_barrier(conf); if (!bio) { set_bit(R10BIO_Returned, &r10_bio->state); goto err_handle; @@ -1231,7 +1210,6 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, r10_bio->master_bio = bio; } read_bio = bio_alloc_clone(rdev->bdev, bio, gfp, &mddev->bio_set); - read_bio->bi_opf &= ~REQ_NOWAIT; r10_bio->devs[slot].bio = read_bio; r10_bio->devs[slot].rdev = rdev; @@ -1265,7 +1243,6 @@ static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio, conf->mirrors[devnum].rdev; mbio = bio_alloc_clone(rdev->bdev, bio, GFP_NOIO, &mddev->bio_set); - mbio->bi_opf &= ~REQ_NOWAIT; if (replacement) r10_bio->devs[n_copy].repl_bio = mbio; else @@ -1344,7 +1321,7 @@ retry_wait: "raid10 %s wait rdev %d blocked", __func__, blocked_rdev->raid_disk); md_wait_for_blocked_rdev(blocked_rdev, mddev); - wait_barrier(conf, false); + wait_barrier(conf); goto retry_wait; } } @@ -1356,33 +1333,20 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio, int i, k; sector_t sectors; int max_sectors; + bool atomic = bio->bi_opf & REQ_ATOMIC; if ((mddev_is_clustered(mddev) && mddev->cluster_ops->area_resyncing(mddev, WRITE, bio->bi_iter.bi_sector, bio_end_sector(bio)))) { - DEFINE_WAIT(w); - /* Bail out if REQ_NOWAIT is set for the bio */ - if (bio->bi_opf & REQ_NOWAIT) { - bio_wouldblock_error(bio); - return false; - } - for (;;) { - prepare_to_wait(&conf->wait_barrier, - &w, TASK_IDLE); - if (!mddev->cluster_ops->area_resyncing(mddev, WRITE, - bio->bi_iter.bi_sector, bio_end_sector(bio))) - break; - schedule(); - } - finish_wait(&conf->wait_barrier, &w); + wait_event_idle(conf->wait_barrier, + !mddev->cluster_ops->area_resyncing(mddev, WRITE, + bio->bi_iter.bi_sector, + bio_end_sector(bio))); } sectors = r10_bio->sectors; - if (!regular_request_wait(mddev, conf, bio, sectors)) { - free_r10bio(r10_bio); - return false; - } + regular_request_wait(mddev, conf, bio, sectors); if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && (mddev->reshape_backwards @@ -1395,11 +1359,6 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio, set_mask_bits(&mddev->sb_flags, 0, BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING)); md_wakeup_thread(mddev->thread); - if (bio->bi_opf & REQ_NOWAIT) { - allow_barrier(conf); - bio_wouldblock_error(bio); - return false; - } mddev_add_trace_msg(conf->mddev, "raid10 wait reshape metadata"); wait_event(mddev->sb_wait, @@ -1462,16 +1421,6 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio, if (is_bad) { int good_sectors; - /* - * We cannot atomically write this, so just - * error in that case. It could be possible to - * atomically write other mirrors, but the - * complexity of supporting that is not worth - * the benefit. - */ - if (bio->bi_opf & REQ_ATOMIC) - goto err_handle; - good_sectors = first_bad - dev_sector; if (good_sectors < max_sectors) max_sectors = good_sectors; @@ -1491,10 +1440,13 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio, r10_bio->sectors = max_sectors; if (r10_bio->sectors < bio_sectors(bio)) { + if (atomic) + goto err_handle; + allow_barrier(conf); bio = bio_submit_split_bioset(bio, r10_bio->sectors, &conf->bio_split); - wait_barrier(conf, false); + wait_barrier(conf); if (!bio) { set_bit(R10BIO_Returned, &r10_bio->state); goto err_handle; @@ -1637,11 +1589,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) return -EAGAIN; - if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) { - bio_wouldblock_error(bio); - md_write_end(mddev); - return 0; - } + wait_barrier(conf); /* * Check reshape again to avoid reshape happens after checking @@ -1692,7 +1640,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) allow_barrier(conf); /* Resend the fist split part */ submit_bio_noacct(split); - wait_barrier(conf, false); + wait_barrier(conf); } div_u64_rem(bio_end, stripe_size, &remainder); if (remainder) { @@ -1712,7 +1660,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) /* Resend the second split part */ submit_bio_noacct(bio); bio = split; - wait_barrier(conf, false); + wait_barrier(conf); } bio_start = bio->bi_iter.bi_sector; @@ -1870,7 +1818,7 @@ retry_discard: end_disk_offset += geo->stride; atomic_inc(&first_r10bio->remaining); raid_end_discard_bio(r10_bio); - wait_barrier(conf, false); + wait_barrier(conf); goto retry_discard; } @@ -2069,7 +2017,7 @@ static void print_conf(struct r10conf *conf) static void close_sync(struct r10conf *conf) { - wait_barrier(conf, false); + wait_barrier(conf); allow_barrier(conf); mempool_exit(&conf->r10buf_pool); @@ -4702,7 +4650,7 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, if (need_flush || time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { /* Need to update reshape_position in metadata */ - wait_barrier(conf, false); + wait_barrier(conf); mddev->reshape_position = conf->reshape_progress; if (mddev->reshape_backwards) mddev->curr_resync_completed = raid10_size(mddev, 0, 0) diff --git a/drivers/md/raid5-ppl.c b/drivers/md/raid5-ppl.c index 7be1648c4e4f0c..7f8a9d3fd578da 100644 --- a/drivers/md/raid5-ppl.c +++ b/drivers/md/raid5-ppl.c @@ -643,8 +643,10 @@ static void ppl_do_flush(struct ppl_io_unit *io) log->disk_flush_bitmap = 0; for (i = flushed_disks ; i < raid_disks; i++) { - if (atomic_dec_and_test(&io->pending_flushes)) + if (atomic_dec_and_test(&io->pending_flushes)) { ppl_io_unit_finished(io); + break; + } } } diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index ffb5fcde54a982..d236a0d772d145 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -801,7 +801,7 @@ static bool is_inactive_blocked(struct r5conf *conf, int hash) return true; return (atomic_read(&conf->active_stripes) < - (conf->max_nr_stripes * 3 / 4)); + (READ_ONCE(conf->max_nr_stripes) * 3 / 4)); } struct stripe_head *raid5_get_active_stripe(struct r5conf *conf, @@ -2471,11 +2471,6 @@ static int scribble_alloc(struct raid5_percpu *percpu, sizeof(unsigned int) * (num + 2); void *scribble; - /* - * If here is in raid array suspend context, it is in memalloc noio - * context as well, there is no potential recursive memory reclaim - * I/Os with the GFP_KERNEL flag. - */ scribble = kvmalloc_array(cnt, obj_size, GFP_KERNEL); if (!scribble) return -ENOMEM; @@ -2490,6 +2485,7 @@ static int scribble_alloc(struct raid5_percpu *percpu, static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) { unsigned long cpu; + unsigned int noio_flags; int err = 0; /* Never shrink. */ @@ -2498,6 +2494,7 @@ static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) return 0; raid5_quiesce(conf->mddev, true); + noio_flags = memalloc_noio_save(); cpus_read_lock(); for_each_present_cpu(cpu) { @@ -2511,6 +2508,7 @@ static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) } cpus_read_unlock(); + memalloc_noio_restore(noio_flags); raid5_quiesce(conf->mddev, false); if (!err) { @@ -2785,6 +2783,7 @@ static void raid5_end_read_request(struct bio * bi) } else { int retry = 0; int set_bad = 0; + int max_nr_stripes = READ_ONCE(conf->max_nr_stripes); clear_bit(R5_UPTODATE, &sh->dev[i].flags); if (!(bi->bi_status == BLK_STS_PROTECTION)) @@ -2810,13 +2809,12 @@ static void raid5_end_read_request(struct bio * bi) mdname(conf->mddev), (unsigned long long)s, rdev->bdev); - } else if (atomic_read(&rdev->read_errors) - > conf->max_nr_stripes) { + } else if (atomic_read(&rdev->read_errors) > max_nr_stripes) { if (!test_bit(Faulty, &rdev->flags)) { pr_warn("md/raid:%s: %d read_errors > %d stripes\n", mdname(conf->mddev), atomic_read(&rdev->read_errors), - conf->max_nr_stripes); + max_nr_stripes); pr_warn("md/raid:%s: Too many read errors, failing device %pg.\n", mdname(conf->mddev), rdev->bdev); } @@ -3553,7 +3551,7 @@ static void __add_stripe_bio(struct stripe_head *sh, struct bio *bi, sh->dev[dd_idx].sector); if (conf->mddev->bitmap && firstwrite && !sh->batch_head) { - sh->bm_seq = conf->seq_flush+1; + sh->bm_seq = READ_ONCE(conf->seq_flush) + 1; set_bit(STRIPE_BIT_DELAY, &sh->state); } } @@ -3751,11 +3749,10 @@ static int want_replace(struct stripe_head *sh, int disk_idx) int rv = 0; rdev = sh->raid_conf->disks[disk_idx].replacement; - if (rdev - && !test_bit(Faulty, &rdev->flags) - && !test_bit(In_sync, &rdev->flags) - && (rdev->recovery_offset <= sh->sector - || rdev->mddev->resync_offset <= sh->sector)) + if (rdev && !test_bit(Faulty, &rdev->flags) && + !test_bit(In_sync, &rdev->flags) && + (READ_ONCE(rdev->recovery_offset) <= sh->sector || + rdev->mddev->resync_offset <= sh->sector)) rv = 1; return rv; } @@ -4672,7 +4669,8 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) */ rdev = conf->disks[i].replacement; if (rdev && !test_bit(Faulty, &rdev->flags) && - rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) && + READ_ONCE(rdev->recovery_offset) >= + sh->sector + RAID5_STRIPE_SECTORS(conf) && !rdev_has_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf))) set_bit(R5_ReadRepl, &dev->flags); @@ -4714,7 +4712,7 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) } else if (test_bit(In_sync, &rdev->flags)) set_bit(R5_Insync, &dev->flags); else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <= - rdev->recovery_offset) { + READ_ONCE(rdev->recovery_offset)) { /* * in sync if: * - normal IO, or @@ -5458,13 +5456,13 @@ static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio) rdev = conf->disks[dd_idx].replacement; if (!rdev || test_bit(Faulty, &rdev->flags) || - rdev->recovery_offset < end_sector) { + READ_ONCE(rdev->recovery_offset) < end_sector) { rdev = conf->disks[dd_idx].rdev; if (!rdev) return 0; if (test_bit(Faulty, &rdev->flags) || !(test_bit(In_sync, &rdev->flags) || - rdev->recovery_offset >= end_sector)) + READ_ONCE(rdev->recovery_offset) >= end_sector)) return 0; } @@ -5718,10 +5716,6 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) struct bio *orig_bi = bi; int stripe_sectors; - /* We need to handle this when io_uring supports discard/trim */ - if (WARN_ON_ONCE(bi->bi_opf & REQ_NOWAIT)) - return; - if (mddev->reshape_position != MaxSector) /* Skip discard while reshape is happening */ return; @@ -5799,7 +5793,7 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) } spin_unlock_irq(&sh->stripe_lock); if (conf->mddev->bitmap) { - sh->bm_seq = conf->seq_flush + 1; + sh->bm_seq = READ_ONCE(conf->seq_flush) + 1; set_bit(STRIPE_BIT_DELAY, &sh->state); } @@ -5956,8 +5950,11 @@ static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset, sectors_per_chunk = conf->chunk_sectors * (conf->raid_disks - conf->max_degraded); - start = round_down(start, sectors_per_chunk); - end = round_up(end, sectors_per_chunk); + sector_div(start, sectors_per_chunk); + start *= sectors_per_chunk; + if (sector_div(end, sectors_per_chunk)) + end++; + end *= sectors_per_chunk; start = raid5_compute_sector(conf, start, 0, &dd_idx, NULL); end = raid5_compute_sector(conf, end, 0, &dd_idx, NULL); @@ -5975,8 +5972,10 @@ static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset, sectors_per_chunk = conf->prev_chunk_sectors * (conf->previous_raid_disks - conf->max_degraded); - prev_start = round_down(prev_start, sectors_per_chunk); - prev_end = round_down(prev_end, sectors_per_chunk); + sector_div(prev_start, sectors_per_chunk); + prev_start *= sectors_per_chunk; + sector_div(prev_end, sectors_per_chunk); + prev_end *= sectors_per_chunk; prev_start = raid5_compute_sector(conf, prev_start, 1, &dd_idx, NULL); prev_end = raid5_compute_sector(conf, prev_end, 1, &dd_idx, NULL); @@ -6191,15 +6190,6 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi) pr_debug("raid456: %s, logical %llu to %llu\n", __func__, bi->bi_iter.bi_sector, ctx->last_sector); - /* Bail out if conflicts with reshape and REQ_NOWAIT is set */ - if ((bi->bi_opf & REQ_NOWAIT) && - get_reshape_loc(mddev, conf, logical_sector) == LOC_INSIDE_RESHAPE) { - bio_wouldblock_error(bi); - if (rw == WRITE) - md_write_end(mddev); - mempool_free(ctx, conf->ctx_pool); - return true; - } md_account_bio(mddev, &bi); /* @@ -6427,8 +6417,8 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk if (rdev->raid_disk >= 0 && !test_bit(Journal, &rdev->flags) && !test_bit(In_sync, &rdev->flags) && - rdev->recovery_offset < sector_nr) - rdev->recovery_offset = sector_nr; + READ_ONCE(rdev->recovery_offset) < sector_nr) + WRITE_ONCE(rdev->recovery_offset, sector_nr); conf->reshape_checkpoint = jiffies; set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); @@ -6536,8 +6526,8 @@ finish: if (rdev->raid_disk >= 0 && !test_bit(Journal, &rdev->flags) && !test_bit(In_sync, &rdev->flags) && - rdev->recovery_offset < sector_nr) - rdev->recovery_offset = sector_nr; + READ_ONCE(rdev->recovery_offset) < sector_nr) + WRITE_ONCE(rdev->recovery_offset, sector_nr); conf->reshape_checkpoint = jiffies; set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); md_wakeup_thread(mddev->thread); @@ -6849,12 +6839,14 @@ static void raid5d(struct md_thread *thread) if ( !list_empty(&conf->bitmap_list)) { /* Now is a good time to flush some bitmap updates */ - conf->seq_flush++; + int seq = conf->seq_flush + 1; + + WRITE_ONCE(conf->seq_flush, seq); spin_unlock_irq(&conf->device_lock); if (md_bitmap_enabled(mddev, true)) mddev->bitmap_ops->unplug(mddev, true); spin_lock_irq(&conf->device_lock); - conf->seq_write = conf->seq_flush; + conf->seq_write = seq; activate_bit_delay(conf, conf->temp_inactive_list); } raid5_activate_delayed(conf); @@ -7039,6 +7031,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len) { struct r5conf *conf; unsigned long new; + unsigned int noio_flags = 0; int err; int size; @@ -7079,6 +7072,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len) goto out_unlock; } + noio_flags = memalloc_noio_save(); mutex_lock(&conf->cache_size_mutex); size = conf->max_nr_stripes; @@ -7095,6 +7089,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len) mutex_unlock(&conf->cache_size_mutex); out_unlock: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return err ?: len; } @@ -8056,9 +8051,9 @@ static int raid5_run(struct mddev *mddev) /* Hack because v0.91 doesn't store recovery_offset properly. */ if (mddev->major_version == 0 && mddev->minor_version > 90) - rdev->recovery_offset = reshape_offset; + WRITE_ONCE(rdev->recovery_offset, reshape_offset); - if (rdev->recovery_offset < reshape_offset) { + if (READ_ONCE(rdev->recovery_offset) < reshape_offset) { /* We need to check old and new layout */ if (!only_parity(rdev->raid_disk, conf->algorithm, @@ -8213,10 +8208,10 @@ static int raid5_spare_active(struct mddev *mddev) for (i = 0; i < conf->raid_disks; i++) { rdev = conf->disks[i].rdev; replacement = conf->disks[i].replacement; - if (replacement - && replacement->recovery_offset == MaxSector - && !test_bit(Faulty, &replacement->flags) - && !test_and_set_bit(In_sync, &replacement->flags)) { + if (replacement && + READ_ONCE(replacement->recovery_offset) == MaxSector && + !test_bit(Faulty, &replacement->flags) && + !test_and_set_bit(In_sync, &replacement->flags)) { /* Replacement has just become active. */ if (!rdev || !test_and_clear_bit(In_sync, &rdev->flags)) @@ -8231,10 +8226,10 @@ static int raid5_spare_active(struct mddev *mddev) rdev->sysfs_state); } sysfs_notify_dirent_safe(replacement->sysfs_state); - } else if (rdev - && rdev->recovery_offset == MaxSector - && !test_bit(Faulty, &rdev->flags) - && !test_and_set_bit(In_sync, &rdev->flags)) { + } else if (rdev && + READ_ONCE(rdev->recovery_offset) == MaxSector && + !test_bit(Faulty, &rdev->flags) && + !test_and_set_bit(In_sync, &rdev->flags)) { count++; sysfs_notify_dirent_safe(rdev->sysfs_state); } @@ -8603,7 +8598,7 @@ static int raid5_start_reshape(struct mddev *mddev) >= conf->previous_raid_disks) set_bit(In_sync, &rdev->flags); else - rdev->recovery_offset = 0; + WRITE_ONCE(rdev->recovery_offset, 0); /* Failure here is OK */ sysfs_link_rdev(mddev, rdev); @@ -8655,7 +8650,7 @@ static void end_reshape(struct r5conf *conf) if (rdev->raid_disk >= 0 && !test_bit(Journal, &rdev->flags) && !test_bit(In_sync, &rdev->flags)) - rdev->recovery_offset = MaxSector; + WRITE_ONCE(rdev->recovery_offset, MaxSector); spin_unlock_irq(&conf->device_lock); wake_up(&conf->wait_for_reshape); @@ -8975,6 +8970,7 @@ static void *raid6_takeover(struct mddev *mddev) static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf) { struct r5conf *conf; + unsigned int noio_flags; int err; err = mddev_suspend_and_lock(mddev); @@ -8986,6 +8982,7 @@ static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf) return -ENODEV; } + noio_flags = memalloc_noio_save(); if (strncmp(buf, "ppl", 3) == 0) { /* ppl only works with RAID 5 */ if (!raid5_has_ppl(conf) && conf->level == 5) { @@ -9025,6 +9022,7 @@ static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf) if (!err) md_update_sb(mddev, 1); + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return err; diff --git a/drivers/media/cec/core/cec-pin.c b/drivers/media/cec/core/cec-pin.c index 6e1c3910283221..085fc12067afa8 100644 --- a/drivers/media/cec/core/cec-pin.c +++ b/drivers/media/cec/core/cec-pin.c @@ -115,7 +115,7 @@ static void cec_pin_update(struct cec_pin *pin, bool v, bool force) return; pin->adap->cec_pin_is_high = v; - if (atomic_read(&pin->work_pin_num_events) < CEC_NUM_PIN_EVENTS) { + if (atomic_read_acquire(&pin->work_pin_num_events) < CEC_NUM_PIN_EVENTS) { u8 ev = v; if (pin->work_pin_events_dropped) { @@ -126,7 +126,7 @@ static void cec_pin_update(struct cec_pin *pin, bool v, bool force) pin->work_pin_ts[pin->work_pin_events_wr] = ktime_get(); pin->work_pin_events_wr = (pin->work_pin_events_wr + 1) % CEC_NUM_PIN_EVENTS; - atomic_inc(&pin->work_pin_num_events); + atomic_inc_return_release(&pin->work_pin_num_events); } else { pin->work_pin_events_dropped = true; pin->work_pin_events_dropped_cnt++; @@ -1101,7 +1101,7 @@ static int cec_pin_thread_func(void *_adap) pin->work_tx_ts); } - while (atomic_read(&pin->work_pin_num_events)) { + while (atomic_read_acquire(&pin->work_pin_num_events)) { unsigned int idx = pin->work_pin_events_rd; u8 v = pin->work_pin_events[idx]; @@ -1110,7 +1110,7 @@ static int cec_pin_thread_func(void *_adap) v & CEC_PIN_EVENT_FL_DROPPED, pin->work_pin_ts[idx]); pin->work_pin_events_rd = (idx + 1) % CEC_NUM_PIN_EVENTS; - atomic_dec(&pin->work_pin_num_events); + atomic_dec_return_release(&pin->work_pin_num_events); } switch (atomic_xchg(&pin->work_irq_change, diff --git a/drivers/media/i2c/rdacm21.c b/drivers/media/i2c/rdacm21.c index bcab462708c702..ece8a410e7cedd 100644 --- a/drivers/media/i2c/rdacm21.c +++ b/drivers/media/i2c/rdacm21.c @@ -588,10 +588,12 @@ static int rdacm21_probe(struct i2c_client *client) ret = v4l2_async_register_subdev(&dev->sd); if (ret) - goto error_free_ctrls; + goto error_entity_cleanup; return 0; +error_entity_cleanup: + media_entity_cleanup(&dev->sd.entity); error_free_ctrls: v4l2_ctrl_handler_free(&dev->ctrls); error: @@ -606,6 +608,7 @@ static void rdacm21_remove(struct i2c_client *client) v4l2_async_unregister_subdev(&dev->sd); v4l2_ctrl_handler_free(&dev->ctrls); + media_entity_cleanup(&dev->sd.entity); i2c_unregister_device(dev->isp); } diff --git a/drivers/media/pci/intel/ipu6/ipu6.c b/drivers/media/pci/intel/ipu6/ipu6.c index d033d461816927..5449a2006bcce7 100644 --- a/drivers/media/pci/intel/ipu6/ipu6.c +++ b/drivers/media/pci/intel/ipu6/ipu6.c @@ -400,7 +400,6 @@ ipu6_isys_init(struct pci_dev *pdev, struct device *parent, &ipdata->hw_variant); if (IS_ERR(isys_adev->mmu)) { put_device(&isys_adev->auxdev.dev); - kfree(pdata); return dev_err_cast_probe(dev, isys_adev->mmu, "ipu6_mmu_init(isys_adev->mmu) failed\n"); } @@ -408,10 +407,8 @@ ipu6_isys_init(struct pci_dev *pdev, struct device *parent, isys_adev->mmu->dev = &isys_adev->auxdev.dev; ret = ipu6_bus_add_device(isys_adev); - if (ret) { - kfree(pdata); + if (ret) return ERR_PTR(ret); - } return isys_adev; } @@ -444,7 +441,6 @@ ipu6_psys_init(struct pci_dev *pdev, struct device *parent, &ipdata->hw_variant); if (IS_ERR(psys_adev->mmu)) { put_device(&psys_adev->auxdev.dev); - kfree(pdata); return dev_err_cast_probe(&pdev->dev, psys_adev->mmu, "ipu6_mmu_init(psys_adev->mmu) failed\n"); } @@ -452,10 +448,8 @@ ipu6_psys_init(struct pci_dev *pdev, struct device *parent, psys_adev->mmu->dev = &psys_adev->auxdev.dev; ret = ipu6_bus_add_device(psys_adev); - if (ret) { - kfree(pdata); + if (ret) return ERR_PTR(ret); - } return psys_adev; } diff --git a/drivers/media/platform/amd/isp4/Kconfig b/drivers/media/platform/amd/isp4/Kconfig index 9d1927af1cb8c1..77b61fae82bab9 100644 --- a/drivers/media/platform/amd/isp4/Kconfig +++ b/drivers/media/platform/amd/isp4/Kconfig @@ -5,6 +5,7 @@ config VIDEO_AMD_ISP4_CAPTURE depends on DRM_AMDGPU && DRM_AMD_ISP depends on HAS_DMA depends on VIDEO_DEV + select MEDIA_CONTROLLER select VIDEOBUF2_CORE select VIDEOBUF2_MEMOPS select VIDEOBUF2_V4L2 diff --git a/drivers/media/platform/amd/isp4/isp4_interface.c b/drivers/media/platform/amd/isp4/isp4_interface.c index 8d73f66bb42cb8..4801617f95599d 100644 --- a/drivers/media/platform/amd/isp4/isp4_interface.c +++ b/drivers/media/platform/amd/isp4/isp4_interface.c @@ -148,12 +148,9 @@ static void isp4if_gpu_mem_free(struct isp4_interface *ispif, struct isp4if_gpu_mem_info **mem_info_ptr) { struct isp4if_gpu_mem_info *mem_info = *mem_info_ptr; - struct device *dev = ispif->dev; - if (!mem_info) { - dev_err(dev, "invalid mem_info\n"); + if (!mem_info) return; - } *mem_info_ptr = NULL; isp_kernel_buffer_free(&mem_info->mem_handle, &mem_info->gpu_mc_addr, @@ -201,6 +198,7 @@ static int isp4if_alloc_fw_gpumem(struct isp4_interface *ispif) error_no_memory: dev_err(dev, "failed to allocate gpu memory\n"); + isp4if_dealloc_fw_gpumem(ispif); return -ENOMEM; } @@ -375,7 +373,7 @@ static int isp4if_send_fw_cmd(struct isp4_interface *ispif, u32 cmd_id, return -ENOMEM; /* Get two references: one for the resp thread, one for us */ - atomic_set(&ele->refcnt, 2); + refcount_set(&ele->refcnt, 2); init_completion(&ele->cmd_done); } @@ -455,7 +453,7 @@ err_dequeue_ele: put_ele_ref: /* Don't free the command if we didn't put the last reference */ - if (ele && atomic_dec_return(&ele->refcnt)) + if (ele && !refcount_dec_and_test(&ele->refcnt)) ele = NULL; free_ele: diff --git a/drivers/media/platform/amd/isp4/isp4_interface.h b/drivers/media/platform/amd/isp4/isp4_interface.h index ce3ac9b9e5cda1..04db71cd54e6ca 100644 --- a/drivers/media/platform/amd/isp4/isp4_interface.h +++ b/drivers/media/platform/amd/isp4/isp4_interface.h @@ -68,7 +68,7 @@ struct isp4if_cmd_element { u32 seq_num; u32 cmd_id; struct completion cmd_done; - atomic_t refcnt; + refcount_t refcnt; }; struct isp4_interface { diff --git a/drivers/media/platform/amd/isp4/isp4_subdev.c b/drivers/media/platform/amd/isp4/isp4_subdev.c index 48deea79ce6c25..6716ab9c128a01 100644 --- a/drivers/media/platform/amd/isp4/isp4_subdev.c +++ b/drivers/media/platform/amd/isp4/isp4_subdev.c @@ -391,7 +391,7 @@ static void isp4sd_fw_resp_cmd_done(struct isp4_subdev *isp_subdev, if (ele) { complete(&ele->cmd_done); - if (atomic_dec_and_test(&ele->refcnt)) + if (refcount_dec_and_test(&ele->refcnt)) kfree(ele); } } @@ -687,7 +687,7 @@ int isp4sd_pwron_and_init(struct v4l2_subdev *sd) if (ret) { dev_err(dev, "fail to power on isp_subdev ret %d\n", ret); - goto err_deinit; + goto err_module_disable; } /* ISPPG ISP Power Status */ @@ -697,7 +697,7 @@ int isp4sd_pwron_and_init(struct v4l2_subdev *sd) dev_err(dev, "fail to set performance state %u, ret %d\n", perf_state, ret); - goto err_deinit; + goto err_power_off; } ispif->status = ISP4IF_STATUS_PWR_ON; @@ -709,12 +709,12 @@ int isp4sd_pwron_and_init(struct v4l2_subdev *sd) ret = isp4if_start(ispif); if (ret) { dev_err(dev, "fail to start isp_subdev interface\n"); - goto err_deinit; + goto err_perf_restore; } if (isp4sd_start_resp_proc_threads(isp_subdev)) { dev_err(dev, "isp_start_resp_proc_threads fail\n"); - goto err_deinit; + goto err_stop_interface; } dev_dbg(dev, "create resp threads ok\n"); @@ -724,8 +724,24 @@ int isp4sd_pwron_and_init(struct v4l2_subdev *sd) isp_subdev->irq_enabled = true; return 0; -err_deinit: - isp4sd_pwroff_and_deinit(sd); + +err_stop_interface: + isp4if_stop(ispif); +err_perf_restore: + ret = dev_pm_genpd_set_performance_state(dev, ISP4SD_PERFORMANCE_STATE_LOW); + if (ret) + dev_err(dev, "fail to set performance state %u, ret %d\n", + ISP4SD_PERFORMANCE_STATE_LOW, ret); +err_power_off: + isp4hw_wreg(isp_subdev->mmio, ISP_SOFT_RESET, 0); + isp4hw_wreg(isp_subdev->mmio, ISP_POWER_STATUS, 0); + ret = pm_runtime_put_sync(dev); + if (ret) + dev_err(dev, "power off isp_subdev fail %d\n", ret); + ispif->status = ISP4IF_STATUS_PWR_OFF; +err_module_disable: + isp4sd_module_enable(isp_subdev, false); + msleep(20); return -EINVAL; } diff --git a/drivers/media/platform/broadcom/bcm2835-unicam.c b/drivers/media/platform/broadcom/bcm2835-unicam.c index cc7627e9a51a83..14bb916dd7b1b1 100644 --- a/drivers/media/platform/broadcom/bcm2835-unicam.c +++ b/drivers/media/platform/broadcom/bcm2835-unicam.c @@ -2614,6 +2614,7 @@ static int unicam_async_nf_init(struct unicam_device *unicam) return 0; error: + v4l2_async_nf_cleanup(&unicam->notifier); fwnode_handle_put(ep_handle); return ret; } @@ -2746,6 +2747,7 @@ static void unicam_remove(struct platform_device *pdev) v4l2_device_unregister(&unicam->v4l2_dev); media_device_unregister(&unicam->mdev); v4l2_async_nf_unregister(&unicam->notifier); + v4l2_async_nf_cleanup(&unicam->notifier); unicam_subdev_cleanup(unicam); diff --git a/drivers/media/platform/qcom/iris/iris_core.c b/drivers/media/platform/qcom/iris/iris_core.c index 52bf56e517f91e..8c335dbfce1668 100644 --- a/drivers/media/platform/qcom/iris/iris_core.c +++ b/drivers/media/platform/qcom/iris/iris_core.c @@ -12,18 +12,24 @@ void iris_core_deinit(struct iris_core *core) { - pm_runtime_resume_and_get(core->dev); + int ret; + + ret = pm_runtime_resume_and_get(core->dev); mutex_lock(&core->lock); if (core->state != IRIS_CORE_DEINIT) { iris_fw_unload(core); - iris_vpu_power_off(core); + + if (!ret) + iris_vpu_power_off(core); + iris_hfi_queues_deinit(core); core->state = IRIS_CORE_DEINIT; } mutex_unlock(&core->lock); - pm_runtime_put_sync(core->dev); + if (!ret) + pm_runtime_put_sync(core->dev); } static int iris_wait_for_system_response(struct iris_core *core) diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c b/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c index ca2954f8bd3ad5..ee2730c293bcde 100644 --- a/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c +++ b/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c @@ -692,7 +692,6 @@ static int iris_hfi_gen2_session_set_config_params(struct iris_inst *inst, u32 p {HFI_PROP_FRAME_RATE, iris_hfi_gen2_set_frame_rate }, {HFI_PROP_AV1_FILM_GRAIN_PRESENT, iris_hfi_gen2_set_film_grain }, {HFI_PROP_AV1_SUPER_BLOCK_ENABLED, iris_hfi_gen2_set_super_block }, - {HFI_PROP_OPB_ENABLE, iris_hfi_gen2_set_opb_enable }, }; if (inst->domain == DECODER) { diff --git a/drivers/media/platform/qcom/iris/iris_state.c b/drivers/media/platform/qcom/iris/iris_state.c index e991f34916ec6e..5552725c614ea2 100644 --- a/drivers/media/platform/qcom/iris/iris_state.c +++ b/drivers/media/platform/qcom/iris/iris_state.c @@ -269,7 +269,7 @@ bool iris_allow_cmd(struct iris_inst *inst, u32 cmd) return true; } else if (cmd == V4L2_DEC_CMD_STOP || cmd == V4L2_ENC_CMD_STOP) { if (vb2_is_streaming(src_q)) - if (inst->sub_state != IRIS_INST_SUB_DRAIN) + if (!(inst->sub_state & IRIS_INST_SUB_DRAIN)) return true; } diff --git a/drivers/media/platform/st/stm32/stm32-dcmi.c b/drivers/media/platform/st/stm32/stm32-dcmi.c index eeb0199864dd14..fc5acd5dbe7aae 100644 --- a/drivers/media/platform/st/stm32/stm32-dcmi.c +++ b/drivers/media/platform/st/stm32/stm32-dcmi.c @@ -2024,8 +2024,10 @@ static int dcmi_probe(struct platform_device *pdev) mdma_chan = dma_request_chan(&pdev->dev, "mdma_tx"); if (IS_ERR(mdma_chan)) { ret = PTR_ERR(mdma_chan); - if (ret != -ENODEV) - return dev_err_probe(&pdev->dev, ret, "Failed to request MDMA channel\n"); + if (ret != -ENODEV) { + dev_err_probe(&pdev->dev, ret, "Failed to request MDMA channel\n"); + goto err_release_chan; + } mdma_chan = NULL; } @@ -2050,6 +2052,7 @@ static int dcmi_probe(struct platform_device *pdev) dcmi->sram_pool = of_gen_pool_get(pdev->dev.of_node, "sram", 0); if (!dcmi->sram_pool) { dev_info(&pdev->dev, "No SRAM pool, can't use MDMA chaining\n"); + ret = -ENOMEM; goto err_dma_slave_config; } @@ -2061,6 +2064,7 @@ static int dcmi_probe(struct platform_device *pdev) &dcmi->sram_dma_buf); if (!dcmi->sram_buf) { dev_err(dcmi->dev, "Failed to allocate from SRAM\n"); + ret = -ENOMEM; goto err_dma_slave_config; } @@ -2206,12 +2210,13 @@ err_device_unregister: err_media_device_cleanup: media_device_cleanup(&dcmi->mdev); err_mdma_slave_config: - if (dcmi->mdma_chan) + if (mdma_chan) gen_pool_free(dcmi->sram_pool, (unsigned long)dcmi->sram_buf, dcmi->sram_buf_size); err_dma_slave_config: - dma_release_channel(dcmi->dma_chan); - if (dcmi->mdma_chan) + if (mdma_chan) dma_release_channel(mdma_chan); +err_release_chan: + dma_release_channel(chan); return ret; } diff --git a/drivers/media/platform/ti/Kconfig b/drivers/media/platform/ti/Kconfig index d0cb05481bd851..1a020b2bbb4f21 100644 --- a/drivers/media/platform/ti/Kconfig +++ b/drivers/media/platform/ti/Kconfig @@ -50,6 +50,7 @@ config VIDEO_TI_VIP select VIDEO_TI_VPDMA select VIDEO_TI_SC select VIDEO_TI_CSC + select V4L2_FWNODE help Driver support for VIP module on certain TI SoC's VIP = Video Input Port. diff --git a/drivers/media/v4l2-core/v4l2-async.c b/drivers/media/v4l2-core/v4l2-async.c index 0aa4265a67829a..460bf3dbbb8826 100644 --- a/drivers/media/v4l2-core/v4l2-async.c +++ b/drivers/media/v4l2-core/v4l2-async.c @@ -897,9 +897,18 @@ void v4l2_async_unregister_subdev(struct v4l2_subdev *sd) sd->subdev_notifier = NULL; if (sd->asc_list.next) { - list_for_each_entry_safe(asc, asc_tmp, &sd->asc_list, - asc_subdev_entry) { - v4l2_async_unbind_subdev_one(asc->notifier, asc); + if (list_empty(&sd->asc_list)) { + /* + * If the sub-device was registered through other means + * than v4l2-async, there are no async connections but + * the sub-device may still well be registered. + * Unregister it now. + */ + v4l2_device_unregister_subdev(sd); + } else { + list_for_each_entry_safe(asc, asc_tmp, &sd->asc_list, + asc_subdev_entry) + v4l2_async_unbind_subdev_one(asc->notifier, asc); } } diff --git a/drivers/mfd/cs42l43.c b/drivers/mfd/cs42l43.c index ed6d93893de04d..af7fee9ee2cf72 100644 --- a/drivers/mfd/cs42l43.c +++ b/drivers/mfd/cs42l43.c @@ -115,14 +115,14 @@ const struct reg_default cs42l43_reg_default[CS42L43_N_DEFAULTS] = { { CS42L43_DECIM_HPF_WNF_CTRL2, 0x00000001 }, { CS42L43_DECIM_HPF_WNF_CTRL3, 0x00000001 }, { CS42L43_DECIM_HPF_WNF_CTRL4, 0x00000001 }, - { CS42L43B_DECIM_HPF_WNF_CTRL5, 0x00000001 }, - { CS42L43B_DECIM_HPF_WNF_CTRL6, 0x00000001 }, { CS42L43_DMIC_PDM_CTRL, 0x00000000 }, { CS42L43_DECIM_VOL_CTRL_CH1_CH2, 0x20122012 }, { CS42L43_DECIM_VOL_CTRL_CH3_CH4, 0x20122012 }, { CS42L43B_DECIM_VOL_CTRL_CH1_CH2, 0x20122012 }, { CS42L43B_DECIM_VOL_CTRL_CH3_CH4, 0x20122012 }, { CS42L43B_DECIM_VOL_CTRL_CH5_CH6, 0x20122012 }, + { CS42L43B_DECIM_HPF_WNF_CTRL5, 0x00000001 }, + { CS42L43B_DECIM_HPF_WNF_CTRL6, 0x00000001 }, { CS42L43_INTP_VOLUME_CTRL1, 0x00000180 }, { CS42L43_INTP_VOLUME_CTRL2, 0x00000180 }, { CS42L43_AMP1_2_VOL_RAMP, 0x00000022 }, @@ -160,10 +160,10 @@ const struct reg_default cs42l43_reg_default[CS42L43_N_DEFAULTS] = { { CS42L43_SWIRE_DP2_CH2_INPUT, 0x00000000 }, { CS42L43_SWIRE_DP3_CH1_INPUT, 0x00000000 }, { CS42L43_SWIRE_DP3_CH2_INPUT, 0x00000000 }, - { CS42L43B_SWIRE_DP3_CH3_INPUT, 0x00000000 }, - { CS42L43B_SWIRE_DP3_CH4_INPUT, 0x00000000 }, { CS42L43_SWIRE_DP4_CH1_INPUT, 0x00000000 }, { CS42L43_SWIRE_DP4_CH2_INPUT, 0x00000000 }, + { CS42L43B_SWIRE_DP3_CH3_INPUT, 0x00000000 }, + { CS42L43B_SWIRE_DP3_CH4_INPUT, 0x00000000 }, { CS42L43B_SWIRE_DP4_CH3_INPUT, 0x00000000 }, { CS42L43B_SWIRE_DP4_CH4_INPUT, 0x00000000 }, { CS42L43_ASRC_INT1_INPUT1, 0x00000000 }, @@ -178,14 +178,10 @@ const struct reg_default cs42l43_reg_default[CS42L43_N_DEFAULTS] = { { CS42L43_ISRC1INT2_INPUT1, 0x00000000 }, { CS42L43_ISRC1DEC1_INPUT1, 0x00000000 }, { CS42L43_ISRC1DEC2_INPUT1, 0x00000000 }, - { CS42L43B_ISRC1DEC3_INPUT1, 0x00000000 }, - { CS42L43B_ISRC1DEC4_INPUT1, 0x00000000 }, { CS42L43_ISRC2INT1_INPUT1, 0x00000000 }, { CS42L43_ISRC2INT2_INPUT1, 0x00000000 }, { CS42L43_ISRC2DEC1_INPUT1, 0x00000000 }, { CS42L43_ISRC2DEC2_INPUT1, 0x00000000 }, - { CS42L43B_ISRC2DEC3_INPUT1, 0x00000000 }, - { CS42L43B_ISRC2DEC4_INPUT1, 0x00000000 }, { CS42L43_EQ1MIX_INPUT1, 0x00800000 }, { CS42L43_EQ1MIX_INPUT2, 0x00800000 }, { CS42L43_EQ1MIX_INPUT3, 0x00800000 }, @@ -212,6 +208,10 @@ const struct reg_default cs42l43_reg_default[CS42L43_N_DEFAULTS] = { { CS42L43_AMP4MIX_INPUT2, 0x00800000 }, { CS42L43_AMP4MIX_INPUT3, 0x00800000 }, { CS42L43_AMP4MIX_INPUT4, 0x00800000 }, + { CS42L43B_ISRC1DEC3_INPUT1, 0x00000000 }, + { CS42L43B_ISRC1DEC4_INPUT1, 0x00000000 }, + { CS42L43B_ISRC2DEC3_INPUT1, 0x00000000 }, + { CS42L43B_ISRC2DEC4_INPUT1, 0x00000000 }, { CS42L43_ASRC_INT_ENABLES, 0x00000100 }, { CS42L43_ASRC_DEC_ENABLES, 0x00000100 }, { CS42L43_PDNCNTL, 0x00000000 }, diff --git a/drivers/mfd/iqs62x.c b/drivers/mfd/iqs62x.c index ee017617d1d1bb..412ae7777f7294 100644 --- a/drivers/mfd/iqs62x.c +++ b/drivers/mfd/iqs62x.c @@ -237,7 +237,7 @@ static int iqs62x_firmware_parse(struct iqs62x_core *iqs62x, fw_rec = (struct iqs62x_fw_rec *)(fw->data + pos); pos += sizeof(*fw_rec); - if (pos + fw_rec->len - 1 > fw->size) { + if (!fw_rec->len || fw_rec->len - 1 > fw->size - pos) { ret = -EINVAL; break; } diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 358feec2d088fc..514cba7dc897ae 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -410,7 +410,7 @@ static int apple_smc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct apple_smc *smc; - u32 count; + __be32 count; int ret; smc = devm_kzalloc(dev, sizeof(*smc), GFP_KERNEL); @@ -461,8 +461,10 @@ static int apple_smc_probe(struct platform_device *pdev) dev_set_drvdata(&pdev->dev, smc); BLOCKING_INIT_NOTIFIER_HEAD(&smc->event_handlers); - ret = apple_smc_read_u32(smc, SMC_KEY(#KEY), &count); - if (ret) + ret = apple_smc_read(smc, SMC_KEY(#KEY), &count, sizeof(count)); + if (ret >= 0 && ret != sizeof(count)) + ret = -EINVAL; + if (ret < 0) return dev_err_probe(smc->dev, ret, "Failed to get key count"); smc->key_count = be32_to_cpu(count); diff --git a/drivers/mfd/rave-sp.c b/drivers/mfd/rave-sp.c index c1b78d127a261e..05d26d92df03ef 100644 --- a/drivers/mfd/rave-sp.c +++ b/drivers/mfd/rave-sp.c @@ -63,6 +63,12 @@ #define RAVE_SP_TX_BUFFER_SIZE \ (RAVE_SP_STX_ETX_SIZE + 2 * RAVE_SP_RX_BUFFER_SIZE) +enum rave_sp_frame_offset { + RAVE_SP_FRAME_CODE_OFFSET, + RAVE_SP_FRAME_ACK_ID_OFFSET, + RAVE_SP_FRAME_DATA_OFFSET, +}; + /** * enum rave_sp_deframer_state - Possible state for de-framer * @@ -352,7 +358,7 @@ int rave_sp_exec(struct rave_sp *sp, int command, ret = 0; u8 ackid; - command = sp->variant->cmd.translate(data[0]); + command = sp->variant->cmd.translate(data[RAVE_SP_FRAME_CODE_OFFSET]); if (command < 0) return command; @@ -366,8 +372,8 @@ int rave_sp_exec(struct rave_sp *sp, sp->reply = &reply; mutex_unlock(&sp->reply_lock); - data[0] = command; - data[1] = ackid; + data[RAVE_SP_FRAME_CODE_OFFSET] = command; + data[RAVE_SP_FRAME_ACK_ID_OFFSET] = ackid; rave_sp_write(sp, data, data_size); @@ -388,16 +394,23 @@ EXPORT_SYMBOL_GPL(rave_sp_exec); static void rave_sp_receive_event(struct rave_sp *sp, const unsigned char *data, size_t length) { - u8 cmd[] = { - [0] = rave_sp_reply_code(data[0]), - [1] = data[1], - }; + unsigned long action; + u8 cmd[RAVE_SP_FRAME_DATA_OFFSET]; + + if (length < RAVE_SP_FRAME_DATA_OFFSET + 1) { + dev_warn(&sp->serdev->dev, "Dropping short event frame\n"); + return; + } + + cmd[RAVE_SP_FRAME_CODE_OFFSET] = + rave_sp_reply_code(data[RAVE_SP_FRAME_CODE_OFFSET]); + cmd[RAVE_SP_FRAME_ACK_ID_OFFSET] = data[RAVE_SP_FRAME_ACK_ID_OFFSET]; rave_sp_write(sp, cmd, sizeof(cmd)); - blocking_notifier_call_chain(&sp->event_notifier_list, - rave_sp_action_pack(data[0], data[2]), - NULL); + action = rave_sp_action_pack(data[RAVE_SP_FRAME_CODE_OFFSET], + data[RAVE_SP_FRAME_DATA_OFFSET]); + blocking_notifier_call_chain(&sp->event_notifier_list, action, NULL); } static void rave_sp_receive_reply(struct rave_sp *sp, @@ -405,27 +418,35 @@ static void rave_sp_receive_reply(struct rave_sp *sp, { struct device *dev = &sp->serdev->dev; struct rave_sp_reply *reply; - const size_t payload_length = length - 2; + size_t payload_length; + + if (length < RAVE_SP_FRAME_DATA_OFFSET) { + dev_warn(dev, "Dropping short reply frame\n"); + return; + } + payload_length = length - RAVE_SP_FRAME_DATA_OFFSET; mutex_lock(&sp->reply_lock); reply = sp->reply; if (reply) { - if (reply->code == data[0] && reply->ackid == data[1] && + if (reply->code == data[RAVE_SP_FRAME_CODE_OFFSET] && + reply->ackid == data[RAVE_SP_FRAME_ACK_ID_OFFSET] && payload_length >= reply->length) { /* * We are relying on memcpy(dst, src, 0) to be a no-op * when handling commands that have a no-payload reply */ - memcpy(reply->data, &data[2], reply->length); + memcpy(reply->data, &data[RAVE_SP_FRAME_DATA_OFFSET], + reply->length); complete(&reply->received); sp->reply = NULL; } else { dev_err(dev, "Ignoring incorrect reply\n"); dev_dbg(dev, "Code: expected = 0x%08x received = 0x%08x\n", - reply->code, data[0]); + reply->code, data[RAVE_SP_FRAME_CODE_OFFSET]); dev_dbg(dev, "ACK ID: expected = 0x%08x received = 0x%08x\n", - reply->ackid, data[1]); + reply->ackid, data[RAVE_SP_FRAME_ACK_ID_OFFSET]); dev_dbg(dev, "Length: expected = %zu received = %zu\n", reply->length, payload_length); } @@ -439,10 +460,10 @@ static void rave_sp_receive_frame(struct rave_sp *sp, size_t length) { const size_t checksum_length = sp->variant->checksum->length; - const size_t payload_length = length - checksum_length; - const u8 *crc_reported = &data[payload_length]; struct device *dev = &sp->serdev->dev; u8 crc_calculated[RAVE_SP_CHECKSUM_SIZE]; + const u8 *crc_reported; + size_t payload_length; if (unlikely(checksum_length > sizeof(crc_calculated))) { dev_warn(dev, "Checksum too long, dropping\n"); @@ -457,6 +478,9 @@ static void rave_sp_receive_frame(struct rave_sp *sp, return; } + payload_length = length - checksum_length; + crc_reported = &data[payload_length]; + sp->variant->checksum->subroutine(data, payload_length, crc_calculated); @@ -465,10 +489,10 @@ static void rave_sp_receive_frame(struct rave_sp *sp, return; } - if (rave_sp_id_is_event(data[0])) - rave_sp_receive_event(sp, data, length); + if (rave_sp_id_is_event(data[RAVE_SP_FRAME_CODE_OFFSET])) + rave_sp_receive_event(sp, data, payload_length); else - rave_sp_receive_reply(sp, data, length); + rave_sp_receive_reply(sp, data, payload_length); } static size_t rave_sp_receive_buf(struct serdev_device *serdev, diff --git a/drivers/misc/ad525x_dpot-i2c.c b/drivers/misc/ad525x_dpot-i2c.c index 469478f7a1d331..896ad61bb9e170 100644 --- a/drivers/misc/ad525x_dpot-i2c.c +++ b/drivers/misc/ad525x_dpot-i2c.c @@ -105,6 +105,7 @@ MODULE_DEVICE_TABLE(i2c, ad_dpot_id); static struct i2c_driver ad_dpot_i2c_driver = { .driver = { .name = "ad_dpot", + .dev_groups = ad_dpot_groups, }, .probe = ad_dpot_i2c_probe, .remove = ad_dpot_i2c_remove, diff --git a/drivers/misc/ad525x_dpot-spi.c b/drivers/misc/ad525x_dpot-spi.c index 263055bda48b7d..1ebe629715a84b 100644 --- a/drivers/misc/ad525x_dpot-spi.c +++ b/drivers/misc/ad525x_dpot-spi.c @@ -131,6 +131,7 @@ MODULE_DEVICE_TABLE(spi, ad_dpot_spi_id); static struct spi_driver ad_dpot_spi_driver = { .driver = { .name = "ad_dpot", + .dev_groups = ad_dpot_groups, }, .probe = ad_dpot_spi_probe, .remove = ad_dpot_spi_remove, diff --git a/drivers/misc/ad525x_dpot.c b/drivers/misc/ad525x_dpot.c index 57bead9fba1bec..a4e22fd4a1072a 100644 --- a/drivers/misc/ad525x_dpot.c +++ b/drivers/misc/ad525x_dpot.c @@ -630,66 +630,132 @@ static struct attribute *ad525x_attributes_commands[] = { NULL }; -static const struct attribute_group ad525x_group_commands = { - .attrs = ad525x_attributes_commands, +static struct attribute *ad525x_attributes[] = { + &dev_attr_rdac0.attr, + &dev_attr_rdac1.attr, + &dev_attr_rdac2.attr, + &dev_attr_rdac3.attr, + &dev_attr_rdac4.attr, + &dev_attr_rdac5.attr, + &dev_attr_eeprom0.attr, + &dev_attr_eeprom1.attr, + &dev_attr_eeprom2.attr, + &dev_attr_eeprom3.attr, + &dev_attr_eeprom4.attr, + &dev_attr_eeprom5.attr, + &dev_attr_tolerance0.attr, + &dev_attr_tolerance1.attr, + &dev_attr_tolerance2.attr, + &dev_attr_tolerance3.attr, + &dev_attr_tolerance4.attr, + &dev_attr_tolerance5.attr, + &dev_attr_otp0.attr, + &dev_attr_otp1.attr, + &dev_attr_otp2.attr, + &dev_attr_otp3.attr, + &dev_attr_otp4.attr, + &dev_attr_otp5.attr, + &dev_attr_otp0en.attr, + &dev_attr_otp1en.attr, + &dev_attr_otp2en.attr, + &dev_attr_otp3en.attr, + &dev_attr_otp4en.attr, + &dev_attr_otp5en.attr, + &dev_attr_inc_all.attr, + &dev_attr_dec_all.attr, + &dev_attr_inc_all_6db.attr, + &dev_attr_dec_all_6db.attr, + NULL }; -static int ad_dpot_add_files(struct device *dev, - unsigned int features, unsigned int rdac) +static int ad525x_attr_index(struct attribute *attr, + const struct attribute * const *attrs) { - int err = sysfs_create_file(&dev->kobj, - dpot_attrib_wipers[rdac]); - if (features & F_CMD_EEP) - err |= sysfs_create_file(&dev->kobj, - dpot_attrib_eeprom[rdac]); - if (features & F_CMD_TOL) - err |= sysfs_create_file(&dev->kobj, - dpot_attrib_tolerance[rdac]); - if (features & F_CMD_OTP) { - err |= sysfs_create_file(&dev->kobj, - dpot_attrib_otp_en[rdac]); - err |= sysfs_create_file(&dev->kobj, - dpot_attrib_otp[rdac]); - } + int i; - if (err) - dev_err(dev, "failed to register sysfs hooks for RDAC%d\n", - rdac); + for (i = 0; attrs[i]; i++) + if (attr == attrs[i]) + return i; - return err; + return -ENOENT; } -static inline void ad_dpot_remove_files(struct device *dev, - unsigned int features, unsigned int rdac) +static bool ad525x_is_command_attr(struct attribute *attr) { - sysfs_remove_file(&dev->kobj, - dpot_attrib_wipers[rdac]); - if (features & F_CMD_EEP) - sysfs_remove_file(&dev->kobj, - dpot_attrib_eeprom[rdac]); - if (features & F_CMD_TOL) - sysfs_remove_file(&dev->kobj, - dpot_attrib_tolerance[rdac]); - if (features & F_CMD_OTP) { - sysfs_remove_file(&dev->kobj, - dpot_attrib_otp_en[rdac]); - sysfs_remove_file(&dev->kobj, - dpot_attrib_otp[rdac]); + int i; + + for (i = 0; ad525x_attributes_commands[i]; i++) { + if (attr == ad525x_attributes_commands[i]) + return true; } + + return false; +} + +static umode_t ad525x_is_visible(struct kobject *kobj, struct attribute *attr, + int n) +{ + struct device *dev = kobj_to_dev(kobj); + struct dpot_data *data = dev_get_drvdata(dev); + int rdac; + + if (!data) + return 0; + + rdac = ad525x_attr_index(attr, dpot_attrib_wipers); + if (rdac >= 0) + return data->wipers & BIT(rdac) ? attr->mode : 0; + + rdac = ad525x_attr_index(attr, dpot_attrib_eeprom); + if (rdac >= 0) + return (data->wipers & BIT(rdac)) && (data->feat & F_CMD_EEP) ? + attr->mode : 0; + + rdac = ad525x_attr_index(attr, dpot_attrib_tolerance); + if (rdac >= 0) + return (data->wipers & BIT(rdac)) && (data->feat & F_CMD_TOL) ? + attr->mode : 0; + + rdac = ad525x_attr_index(attr, dpot_attrib_otp); + if (rdac >= 0) + return (data->wipers & BIT(rdac)) && (data->feat & F_CMD_OTP) ? + attr->mode : 0; + + rdac = ad525x_attr_index(attr, dpot_attrib_otp_en); + if (rdac >= 0) + return (data->wipers & BIT(rdac)) && (data->feat & F_CMD_OTP) ? + attr->mode : 0; + + if (ad525x_is_command_attr(attr)) + return data->feat & F_CMD_INC ? attr->mode : 0; + + return attr->mode; } +static const struct attribute_group ad525x_group = { + .attrs = ad525x_attributes, + .is_visible = ad525x_is_visible, +}; + +const struct attribute_group *ad_dpot_groups[] = { + &ad525x_group, + NULL +}; +EXPORT_SYMBOL(ad_dpot_groups); + int ad_dpot_probe(struct device *dev, struct ad_dpot_bus_data *bdata, unsigned long devid, const char *name) { struct dpot_data *data; - int i, err = 0; + int i; data = kzalloc_obj(struct dpot_data); if (!data) { - err = -ENOMEM; - goto exit; + dev_err(dev, "failed to create client for %s ID 0x%lX\n", + name, devid); + return -ENOMEM; } dev_set_drvdata(dev, data); @@ -705,51 +771,22 @@ int ad_dpot_probe(struct device *dev, data->wipers = DPOT_WIPERS(devid); for (i = DPOT_RDAC0; i < MAX_RDACS; i++) - if (data->wipers & (1 << i)) { - err = ad_dpot_add_files(dev, data->feat, i); - if (err) - goto exit_remove_files; + if (data->wipers & BIT(i)) { /* power-up midscale */ if (data->feat & F_RDACS_WONLY) data->rdac_cache[i] = data->max_pos / 2; } - if (data->feat & F_CMD_INC) - err = sysfs_create_group(&dev->kobj, &ad525x_group_commands); - - if (err) { - dev_err(dev, "failed to register sysfs hooks\n"); - goto exit_free; - } - dev_info(dev, "%s %d-Position Digital Potentiometer registered\n", name, data->max_pos); return 0; - -exit_remove_files: - for (i = DPOT_RDAC0; i < MAX_RDACS; i++) - if (data->wipers & (1 << i)) - ad_dpot_remove_files(dev, data->feat, i); - -exit_free: - kfree(data); - dev_set_drvdata(dev, NULL); -exit: - dev_err(dev, "failed to create client for %s ID 0x%lX\n", - name, devid); - return err; } EXPORT_SYMBOL(ad_dpot_probe); void ad_dpot_remove(struct device *dev) { struct dpot_data *data = dev_get_drvdata(dev); - int i; - - for (i = DPOT_RDAC0; i < MAX_RDACS; i++) - if (data->wipers & (1 << i)) - ad_dpot_remove_files(dev, data->feat, i); kfree(data); } diff --git a/drivers/misc/ad525x_dpot.h b/drivers/misc/ad525x_dpot.h index 72a9d6801937ca..2e877c89523b5f 100644 --- a/drivers/misc/ad525x_dpot.h +++ b/drivers/misc/ad525x_dpot.h @@ -10,6 +10,8 @@ #include +struct attribute_group; + #define DPOT_CONF(features, wipers, max_pos, uid) \ (((features) << 18) | (((wipers) & 0xFF) << 10) | \ ((max_pos & 0xF) << 6) | (uid & 0x3F)) @@ -210,5 +212,6 @@ struct ad_dpot_bus_data { int ad_dpot_probe(struct device *dev, struct ad_dpot_bus_data *bdata, unsigned long devid, const char *name); void ad_dpot_remove(struct device *dev); +extern const struct attribute_group *ad_dpot_groups[]; #endif diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.c b/drivers/misc/bcm-vk/bcm_vk_msg.c index 3916ec07ecad18..2c084a6b3a9297 100644 --- a/drivers/misc/bcm-vk/bcm_vk_msg.c +++ b/drivers/misc/bcm-vk/bcm_vk_msg.c @@ -108,7 +108,8 @@ u32 msgq_avail_space(const struct bcm_vk_msgq __iomem *msgq, bool bcm_vk_drv_access_ok(struct bcm_vk *vk) { - return (!!atomic_read(&vk->msgq_inited)); + /* Pair with the release store after message queue initialization. */ + return !!atomic_read_acquire(&vk->msgq_inited); } void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask) @@ -501,7 +502,8 @@ int bcm_vk_sync_msgq(struct bcm_vk *vk, bool force_sync) msgq++; } } - atomic_set(&vk->msgq_inited, 1); + /* Publish message queue info before allowing driver access. */ + atomic_set_release(&vk->msgq_inited, 1); return ret; } diff --git a/drivers/misc/cardreader/rtsx_pcr.c b/drivers/misc/cardreader/rtsx_pcr.c index c4d54ca2fa8047..c6e602523538e9 100644 --- a/drivers/misc/cardreader/rtsx_pcr.c +++ b/drivers/misc/cardreader/rtsx_pcr.c @@ -1196,6 +1196,8 @@ static int rtsx_pci_init_hw(struct rtsx_pcr *pcr) /* Gating real mcu clock */ err = rtsx_pci_write_register(pcr, RTS5261_FW_CFG1, RTS5261_MCU_CLOCK_GATING, 0); + if (err < 0) + return err; err = rtsx_pci_write_register(pcr, RTS5261_REG_FPDCTL, SSC_POWER_DOWN, 0); } else { diff --git a/drivers/misc/cardreader/rtsx_usb.c b/drivers/misc/cardreader/rtsx_usb.c index 1830e9ed252165..a127744918f426 100644 --- a/drivers/misc/cardreader/rtsx_usb.c +++ b/drivers/misc/cardreader/rtsx_usb.c @@ -312,6 +312,9 @@ int rtsx_usb_get_card_status(struct rtsx_ucr *ucr, u16 *status) if (ret < 0) return ret; + ucr->card_status_cache = *status; + ucr->card_status_valid = true; + return 0; } EXPORT_SYMBOL_GPL(rtsx_usb_get_card_status); @@ -623,6 +626,7 @@ static int rtsx_usb_probe(struct usb_interface *intf, { struct usb_device *usb_dev = interface_to_usbdev(intf); struct rtsx_ucr *ucr; + u16 status; int ret; dev_dbg(&intf->dev, @@ -659,6 +663,9 @@ static int rtsx_usb_probe(struct usb_interface *intf, if (ret) goto out_init_fail; + /* Prime cached status for runtime autosuspend decisions. */ + rtsx_usb_get_card_status(ucr, &status); + /* initialize USB SG transfer timer */ timer_setup(&ucr->sg_timer, rtsx_usb_sg_timed_out, 0); @@ -713,22 +720,29 @@ static int rtsx_usb_suspend(struct usb_interface *intf, pm_message_t message) struct rtsx_ucr *ucr = (struct rtsx_ucr *)usb_get_intfdata(intf); u16 val = 0; + bool valid = false; dev_dbg(&intf->dev, "%s called with pm message 0x%04x\n", __func__, message.event); if (PMSG_IS_AUTO(message)) { if (mutex_trylock(&ucr->dev_mutex)) { - rtsx_usb_get_card_status(ucr, &val); + valid = ucr->card_status_valid; + if (valid) + val = ucr->card_status_cache; mutex_unlock(&ucr->dev_mutex); - /* Defer the autosuspend if card exists */ - if (val & (SD_CD | MS_CD)) { + /* + * Do not issue USB commands from runtime autosuspend. + * Raw SD_CD is not authoritative on tray-based readers, + * while a real SD card is protected by the SD/MMC child + * runtime-PM reference once the card is powered. Keep + * the historical Memory Stick autosuspend deferral when + * the cached status says MS media is present. + */ + if (valid && (val & MS_CD)) { device_for_each_child(&intf->dev, NULL, rtsx_usb_resume_child); return -EAGAIN; - } else { - /* if the card does not exists, clear OCP status */ - rtsx_usb_write_register(ucr, OCPCTL, MS_OCP_CLEAR, MS_OCP_CLEAR); } } else { /* There is an ongoing operation*/ diff --git a/drivers/misc/lan966x_pci.c b/drivers/misc/lan966x_pci.c index 0bb90c0943bf7f..b0949c653e5ba1 100644 --- a/drivers/misc/lan966x_pci.c +++ b/drivers/misc/lan966x_pci.c @@ -183,6 +183,7 @@ static int lan966x_pci_probe(struct pci_dev *pdev, const struct pci_device_id *i return 0; err_unload_overlay: + of_platform_depopulate(dev); lan966x_pci_unload_overlay(data); return ret; } diff --git a/drivers/misc/mei/mei_lb.c b/drivers/misc/mei/mei_lb.c index f6a258c2b838b7..9fa69acf28d5b0 100644 --- a/drivers/misc/mei/mei_lb.c +++ b/drivers/misc/mei/mei_lb.c @@ -281,7 +281,7 @@ static int mei_lb_check_response_v2(const struct device *dev, ssize_t bytes, bytes, sizeof(rsp->rheader)); return -ENOMSG; } - if (rsp->rheader.header.command_id != MEI_LB2_CMD) { + if (rsp->rheader.header.command_id != cpu_to_le32(MEI_LB2_CMD)) { dev_err(dev, "Mismatch command: 0x%x instead of 0x%x\n", rsp->rheader.header.command_id, MEI_LB2_CMD); return -EPROTO; @@ -327,7 +327,7 @@ static int mei_lb_push_payload_v2(struct device *dev, struct mei_cl_device *clde if (sent_data + chunk_size == payload_size) last_chunk = MEI_LB2_FLAG_LST_CHUNK; - req->header.command_id = MEI_LB2_CMD; + req->header.command_id = cpu_to_le32(MEI_LB2_CMD); req->type = cpu_to_le32(type); req->flags = cpu_to_le32(flags | first_chunk | last_chunk); req->reserved = 0; diff --git a/drivers/misc/pch_phub.c b/drivers/misc/pch_phub.c index 19c4fa017f243e..0097611b97af62 100644 --- a/drivers/misc/pch_phub.c +++ b/drivers/misc/pch_phub.c @@ -83,6 +83,14 @@ #define PCH_PHUB_OROM_SIZE 15360 +enum pch_phub_type { + PCH_EG20T, + PCH_ML7213, + PCH_ML7223M, + PCH_ML7223N, + PCH_ML7831, +}; + /** * struct pch_phub_reg - PHUB register structure * @phub_id_reg: PHUB_ID register val @@ -125,7 +133,7 @@ struct pch_phub_reg { void __iomem *pch_phub_extrom_base_address; u32 pch_mac_start_address; u32 pch_opt_rom_start_address; - int ioh_type; + enum pch_phub_type ioh_type; struct pci_dev *pdev; }; @@ -344,7 +352,7 @@ static int pch_phub_write_gbe_mac_addr(struct pch_phub_reg *chip, u8 *data) int retval; int i; - if ((chip->ioh_type == 1) || (chip->ioh_type == 5)) /* EG20T or ML7831*/ + if (chip->ioh_type == PCH_EG20T || chip->ioh_type == PCH_ML7831) retval = pch_phub_gbe_serial_rom_conf(chip); else /* ML7223 */ retval = pch_phub_gbe_serial_rom_conf_mp(chip); @@ -537,14 +545,6 @@ static const struct bin_attribute pch_bin_attr = { .write = pch_phub_bin_write, }; -enum { - PCH_EG20T, - PCH_ML7213, - PCH_ML7223M, - PCH_ML7223N, - PCH_ML7831, -}; - static int pch_phub_probe(struct pci_dev *pdev, const struct pci_device_id *id) { diff --git a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c index 3635741c3e7a71..26ab9252251f11 100644 --- a/drivers/misc/pci_endpoint_test.c +++ b/drivers/misc/pci_endpoint_test.c @@ -1071,6 +1071,7 @@ static int pci_endpoint_test_doorbell(struct pci_endpoint_test *test) struct pci_dev *pdev = test->pdev; struct device *dev = &pdev->dev; int irq_type = test->irq_type; + int ret = 0; enum pci_barno bar; u32 data, status; u32 addr; @@ -1119,8 +1120,11 @@ static int pci_endpoint_test_doorbell(struct pci_endpoint_test *test) status = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_STATUS); - if (!left || !(status & STATUS_DOORBELL_SUCCESS)) + if (!left || !(status & STATUS_DOORBELL_SUCCESS)) { dev_err(dev, "Failed to trigger doorbell in endpoint\n"); + /* Store error code, but continue to disable doorbell. */ + ret = -EINVAL; + } pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND, COMMAND_DISABLE_DOORBELL); @@ -1134,10 +1138,7 @@ static int pci_endpoint_test_doorbell(struct pci_endpoint_test *test) return -EINVAL; } - if (!(status & STATUS_DOORBELL_SUCCESS)) - return -EINVAL; - - return 0; + return ret; } static long pci_endpoint_test_ioctl(struct file *file, unsigned int cmd, diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index 3557d78ee47a27..5a87c12f444a32 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -166,13 +166,8 @@ static void get_clear_fault_map(struct gru_state *gru, } /* - * Atomic (interrupt context) & non-atomic (user context) functions to - * convert a vaddr into a physical address. The size of the page - * is returned in pageshift. - * returns: - * 0 - successful - * < 0 - error code - * 1 - (atomic only) try again in non-atomic context + * Convert a user virtual address to a physical address in process context. + * The size of the page is returned in pageshift. */ static int non_atomic_pte_lookup(struct vm_area_struct *vma, unsigned long vaddr, int write, @@ -192,87 +187,25 @@ static int non_atomic_pte_lookup(struct vm_area_struct *vma, return 0; } -/* - * atomic_pte_lookup - * - * Convert a user virtual address to a physical address - * Only supports Intel large pages (2MB only) on x86_64. - * ZZZ - hugepage support is incomplete - * - * NOTE: mmap_lock is already held on entry to this function. This - * guarantees existence of the page tables. - */ -static int atomic_pte_lookup(struct vm_area_struct *vma, unsigned long vaddr, - int write, unsigned long *paddr, int *pageshift) -{ - pgd_t *pgdp; - p4d_t *p4dp; - pud_t *pudp; - pmd_t *pmdp; - pte_t pte; - - pgdp = pgd_offset(vma->vm_mm, vaddr); - if (unlikely(pgd_none(*pgdp))) - goto err; - - p4dp = p4d_offset(pgdp, vaddr); - if (unlikely(p4d_none(*p4dp))) - goto err; - - pudp = pud_offset(p4dp, vaddr); - if (unlikely(pud_none(*pudp))) - goto err; - - pmdp = pmd_offset(pudp, vaddr); - if (unlikely(pmd_none(*pmdp))) - goto err; -#ifdef CONFIG_X86_64 - if (unlikely(pmd_leaf(*pmdp))) - pte = ptep_get((pte_t *)pmdp); - else -#endif - pte = *pte_offset_kernel(pmdp, vaddr); - - if (unlikely(!pte_present(pte) || - (write && (!pte_write(pte) || !pte_dirty(pte))))) - return 1; - - *paddr = pte_pfn(pte) << PAGE_SHIFT; -#ifdef CONFIG_HUGETLB_PAGE - *pageshift = is_vm_hugetlb_page(vma) ? HPAGE_SHIFT : PAGE_SHIFT; -#else - *pageshift = PAGE_SHIFT; -#endif - return 0; - -err: - return 1; -} - static int gru_vtop(struct gru_thread_state *gts, unsigned long vaddr, int write, int atomic, unsigned long *gpa, int *pageshift) { struct mm_struct *mm = gts->ts_mm; struct vm_area_struct *vma; unsigned long paddr; - int ret, ps; + int ps; vma = find_vma(mm, vaddr); if (!vma) goto inval; - /* - * Atomic lookup is faster & usually works even if called in non-atomic - * context. - */ - rmb(); /* Must/check ms_range_active before loading PTEs */ - ret = atomic_pte_lookup(vma, vaddr, write, &paddr, &ps); - if (ret) { - if (atomic) - goto upm; - if (non_atomic_pte_lookup(vma, vaddr, write, &paddr, &ps)) - goto inval; - } + if (atomic) + goto upm; + + /* Order the caller's ms_range_active check before loading PTEs. */ + rmb(); + if (non_atomic_pte_lookup(vma, vaddr, write, &paddr, &ps)) + goto inval; if (is_gru_paddr(paddr)) goto inval; paddr = paddr & ~((1UL << ps) - 1); @@ -569,19 +502,9 @@ static irqreturn_t gru_intr(int chiplet, int blade) continue; } - /* - * This is running in interrupt context. Trylock the mmap_lock. - * If it fails, retry the fault in user context. - */ + /* Address translation may sleep, so retry the fault in user context. */ gts->ustats.fmm_tlbmiss++; - if (!gts->ts_force_cch_reload && - mmap_read_trylock(gts->ts_mm)) { - gru_try_dropin(gru, gts, tfh, NULL); - mmap_read_unlock(gts->ts_mm); - } else { - tfh_user_polling_mode(tfh); - STAT(intr_mm_lock_failed); - } + tfh_user_polling_mode(tfh); } return IRQ_HANDLED; } diff --git a/drivers/misc/sgi-gru/gruprocfs.c b/drivers/misc/sgi-gru/gruprocfs.c index 97b8b38ab47dfd..b8139c27bc7f8a 100644 --- a/drivers/misc/sgi-gru/gruprocfs.c +++ b/drivers/misc/sgi-gru/gruprocfs.c @@ -54,7 +54,6 @@ static int statistics_show(struct seq_file *s, void *p) printstat(s, intr_cbr); printstat(s, intr_tfh); printstat(s, intr_spurious); - printstat(s, intr_mm_lock_failed); printstat(s, call_os); printstat(s, call_os_wait_queue); printstat(s, user_flush_tlb); diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h index 640daf1994df71..3348552925c61e 100644 --- a/drivers/misc/sgi-gru/grutables.h +++ b/drivers/misc/sgi-gru/grutables.h @@ -182,7 +182,6 @@ struct gru_stats_s { atomic_long_t intr_cbr; atomic_long_t intr_tfh; atomic_long_t intr_spurious; - atomic_long_t intr_mm_lock_failed; atomic_long_t call_os; atomic_long_t call_os_wait_queue; atomic_long_t user_flush_tlb; diff --git a/drivers/misc/vmw_vmci/vmci_event.c b/drivers/misc/vmw_vmci/vmci_event.c index fffe068a26eb42..c3ef3b98c432e0 100644 --- a/drivers/misc/vmw_vmci/vmci_event.c +++ b/drivers/misc/vmw_vmci/vmci_event.c @@ -179,16 +179,16 @@ int vmci_event_subscribe(u32 event, } } + *new_subscription_id = sub->id; if (have_new_id) { list_add_rcu(&sub->node, &subscriber_array[event]); retval = VMCI_SUCCESS; } else { + kfree(sub); retval = VMCI_ERROR_NO_RESOURCES; } mutex_unlock(&subscriber_mutex); - - *new_subscription_id = sub->id; return retval; } EXPORT_SYMBOL_GPL(vmci_event_subscribe); diff --git a/drivers/mtd/devices/mtd_intel_dg.c b/drivers/mtd/devices/mtd_intel_dg.c index f2fa8f68d19053..a566e86eb5e3a0 100644 --- a/drivers/mtd/devices/mtd_intel_dg.c +++ b/drivers/mtd/devices/mtd_intel_dg.c @@ -780,7 +780,7 @@ static int intel_dg_mtd_probe(struct auxiliary_device *aux_dev, dev_name(&aux_dev->dev), invm->regions[i].name); if (!name) { ret = -ENOMEM; - goto err; + goto err_norpm; } nvm->regions[n].name = name; diff --git a/drivers/mtd/mtdpart.c b/drivers/mtd/mtdpart.c index 4b41550fd374e4..ddded0dbe77e0d 100644 --- a/drivers/mtd/mtdpart.c +++ b/drivers/mtd/mtdpart.c @@ -258,7 +258,8 @@ int mtd_add_partition(struct mtd_info *parent, const char *name, /* the direct offset is expected */ if (offset == MTDPART_OFS_APPEND || - offset == MTDPART_OFS_NXTBLK) + offset == MTDPART_OFS_NXTBLK || + offset == MTDPART_OFS_RETAIN) return -EINVAL; if (length == MTDPART_SIZ_FULL) diff --git a/drivers/mtd/mtdswap.c b/drivers/mtd/mtdswap.c index f33f753f0a9fda..92e38ece893178 100644 --- a/drivers/mtd/mtdswap.c +++ b/drivers/mtd/mtdswap.c @@ -1452,6 +1452,7 @@ static void mtdswap_add_mtd(struct mtd_blktrans_ops *tr, struct mtd_info *mtd) debugfs_failed: del_mtd_blktrans_dev(mbd_dev); + mbd_dev = NULL; cleanup: mtdswap_cleanup(d); diff --git a/drivers/mtd/nand/raw/pl35x-nand-controller.c b/drivers/mtd/nand/raw/pl35x-nand-controller.c index 7d43506b1654d5..12b9e0936c8f16 100644 --- a/drivers/mtd/nand/raw/pl35x-nand-controller.c +++ b/drivers/mtd/nand/raw/pl35x-nand-controller.c @@ -972,17 +972,19 @@ static int pl35x_nand_attach_chip(struct nand_chip *chip) switch (chip->ecc.engine_type) { case NAND_ECC_ENGINE_TYPE_ON_DIE: - dev_dbg(nfc->dev, "Using on-die ECC\n"); + dev_dbg(nfc->dev, "Using on-die hardware ECC\n"); /* Keep these legacy BBT descriptors for ON_DIE situations */ chip->bbt_td = &bbt_main_descr; chip->bbt_md = &bbt_mirror_descr; fallthrough; case NAND_ECC_ENGINE_TYPE_NONE: + dev_dbg(nfc->dev, "Using no ECC engine\n"); + break; case NAND_ECC_ENGINE_TYPE_SOFT: - dev_dbg(nfc->dev, "Using software ECC (Hamming 1-bit/512B)\n"); + dev_dbg(nfc->dev, "Using software ECC\n"); break; case NAND_ECC_ENGINE_TYPE_ON_HOST: - dev_dbg(nfc->dev, "Using hardware ECC\n"); + dev_dbg(nfc->dev, "Using on-host hardware ECC\n"); ret = pl35x_nand_init_hw_ecc_controller(nfc, chip); if (ret) return ret; diff --git a/drivers/mtd/ubi/attach.c b/drivers/mtd/ubi/attach.c index 0fa115cbf3ad43..0ce7ff7400d379 100644 --- a/drivers/mtd/ubi/attach.c +++ b/drivers/mtd/ubi/attach.c @@ -771,7 +771,7 @@ void ubi_remove_av(struct ubi_attach_info *ai, struct ubi_ainf_volume *av) static int early_erase_peb(struct ubi_device *ubi, const struct ubi_attach_info *ai, int pnum, int ec) { - int err; + int err, torture = 0; struct ubi_ec_hdr *ec_hdr; if ((long long)ec >= UBI_MAX_ERASECOUNTER) { @@ -790,7 +790,7 @@ static int early_erase_peb(struct ubi_device *ubi, ec_hdr->ec = cpu_to_be64(ec); - err = ubi_io_sync_erase(ubi, pnum, 0); + err = ubi_io_sync_erase(ubi, pnum, &torture); if (err < 0) goto out_free; diff --git a/drivers/mtd/ubi/build.c b/drivers/mtd/ubi/build.c index 674ad87809df09..456090a12f52bb 100644 --- a/drivers/mtd/ubi/build.c +++ b/drivers/mtd/ubi/build.c @@ -1105,6 +1105,7 @@ int ubi_detach_mtd_dev(int ubi_num, int anyway) ubi->ref_count -= 1; if (ubi->ref_count) { if (!anyway) { + put_device(&ubi->dev); spin_unlock(&ubi_devices_lock); return -EBUSY; } @@ -1317,10 +1318,10 @@ static int __init ubi_init_attach(void) return 0; out_detach: - for (k = 0; k < i; k++) + for (k = 0; k < UBI_MAX_DEVICES; k++) if (ubi_devices[k]) { mutex_lock(&ubi_devices_mutex); - ubi_detach_mtd_dev(ubi_devices[k]->ubi_num, 1); + ubi_detach_mtd_dev(k, 1); mutex_unlock(&ubi_devices_mutex); } return err; @@ -1574,7 +1575,7 @@ static int ubi_mtd_param_parse(const char *val, const struct kernel_param *kp) } module_param_call(mtd, ubi_mtd_param_parse, NULL, NULL, 0400); -MODULE_PARM_DESC(mtd, "MTD devices to attach. Parameter format: mtd=[,[,max_beb_per1024[,ubi_num]]].\n" +MODULE_PARM_DESC(mtd, "MTD devices to attach. Parameter format: mtd=[,[,max_beb_per1024[,ubi_num[,enable_fm[,need_resv_pool]]]]].\n" "Multiple \"mtd\" parameters may be specified.\n" "MTD devices may be specified by their number, name, or path to the MTD character device node.\n" "Optional \"vid_hdr_offs\" parameter specifies UBI VID header position to be used by UBI. (default value if 0)\n" @@ -1587,9 +1588,9 @@ MODULE_PARM_DESC(mtd, "MTD devices to attach. Parameter format: mtd=e[i]) { - ret = ubi_sync_erase(ubi, old_fm->e[i], 0); + ret = ubi_sync_erase(ubi, old_fm->e[i], + &old_fm->to_be_tortured[i]); if (ret < 0) { ubi_err(ubi, "could not erase old fastmap PEB"); @@ -1578,7 +1579,8 @@ int ubi_update_fastmap(struct ubi_device *ubi) if (old_fm) { /* no fresh anchor PEB was found, reuse the old one */ if (!tmp_e) { - ret = ubi_sync_erase(ubi, old_fm->e[0], 0); + ret = ubi_sync_erase(ubi, old_fm->e[0], + &old_fm->to_be_tortured[0]); if (ret < 0) { ubi_err(ubi, "could not erase old anchor PEB"); diff --git a/drivers/mtd/ubi/io.c b/drivers/mtd/ubi/io.c index 915eb64cb00113..d70d1313ad4e80 100644 --- a/drivers/mtd/ubi/io.c +++ b/drivers/mtd/ubi/io.c @@ -519,8 +519,10 @@ error: /** * ubi_io_sync_erase - synchronously erase a physical eraseblock. * @ubi: UBI device description object - * @pnum: physical eraseblock number to erase - * @torture: if this physical eraseblock has to be tortured + * @pnum: the physical eraseblock number to erase + * @torture: if this physical eraseblock has to be tortured; cleared to zero + * once the torture test has completed successfully so that a retry + * of the erase does not torture the physical eraseblock again * * This function synchronously erases physical eraseblock @pnum. If @torture * flag is not zero, the physical eraseblock is checked by means of writing @@ -532,7 +534,7 @@ error: * codes in case of other errors. Note, %-EIO means that the physical * eraseblock is bad. */ -int ubi_io_sync_erase(struct ubi_device *ubi, int pnum, int torture) +int ubi_io_sync_erase(struct ubi_device *ubi, int pnum, int *torture) { int err, ret = 0; @@ -560,10 +562,11 @@ int ubi_io_sync_erase(struct ubi_device *ubi, int pnum, int torture) return err; } - if (torture) { + if (*torture) { ret = torture_peb(ubi, pnum); if (ret < 0) return ret; + *torture = 0; } err = do_sync_erase(ubi, pnum); diff --git a/drivers/mtd/ubi/ubi.h b/drivers/mtd/ubi/ubi.h index af466cd83ae079..80b7628929228b 100644 --- a/drivers/mtd/ubi/ubi.h +++ b/drivers/mtd/ubi/ubi.h @@ -901,7 +901,7 @@ int self_check_eba(struct ubi_device *ubi, struct ubi_attach_info *ai_fastmap, struct ubi_attach_info *ai_scan); /* wl.c */ -int ubi_sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, int torture); +int ubi_sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, int *torture); int ubi_wl_get_peb(struct ubi_device *ubi); int ubi_wl_put_peb(struct ubi_device *ubi, int vol_id, int lnum, int pnum, int torture); @@ -923,7 +923,7 @@ int ubi_io_read(const struct ubi_device *ubi, void *buf, int pnum, int offset, int len); int ubi_io_write(struct ubi_device *ubi, const void *buf, int pnum, int offset, int len); -int ubi_io_sync_erase(struct ubi_device *ubi, int pnum, int torture); +int ubi_io_sync_erase(struct ubi_device *ubi, int pnum, int *torture); int ubi_io_is_bad(const struct ubi_device *ubi, int pnum); int ubi_io_mark_bad(const struct ubi_device *ubi, int pnum); int ubi_io_read_ec_hdr(struct ubi_device *ubi, int pnum, diff --git a/drivers/mtd/ubi/wl.c b/drivers/mtd/ubi/wl.c index e3705db8e57091..edfab98f7064fa 100644 --- a/drivers/mtd/ubi/wl.c +++ b/drivers/mtd/ubi/wl.c @@ -443,12 +443,14 @@ static int prot_queue_del(struct ubi_device *ubi, int pnum) * ubi_sync_erase - synchronously erase a physical eraseblock. * @ubi: UBI device description object * @e: the physical eraseblock to erase - * @torture: if the physical eraseblock has to be tortured + * @torture: if the physical eraseblock has to be tortured; cleared to zero + * once the torture test has completed successfully so that a retry + * of the erase does not torture the physical eraseblock again * * This function returns zero in case of success and a negative error code in * case of failure. */ -int ubi_sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, int torture) +int ubi_sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, int *torture) { int err; struct ubi_ec_hdr *ec_hdr; @@ -1113,7 +1115,7 @@ static int __erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk) dbg_wl("erase PEB %d EC %d LEB %d:%d", pnum, e->ec, wl_wrk->vol_id, wl_wrk->lnum); - err = ubi_sync_erase(ubi, e, wl_wrk->torture); + err = ubi_sync_erase(ubi, e, &wl_wrk->torture); if (!err) { spin_lock(&ubi->wl_lock); @@ -1150,7 +1152,8 @@ static int __erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk) int err1; /* Re-schedule the LEB for erasure */ - err1 = schedule_erase(ubi, e, vol_id, lnum, 0, true); + err1 = schedule_erase(ubi, e, vol_id, lnum, wl_wrk->torture, + true); if (err1) { spin_lock(&ubi->wl_lock); wl_entry_destroy(ubi, e); @@ -1757,7 +1760,7 @@ static void shutdown_work(struct ubi_device *ubi) static int erase_aeb(struct ubi_device *ubi, struct ubi_ainf_peb *aeb, bool sync) { struct ubi_wl_entry *e; - int err; + int err, torture = 0; e = kmem_cache_alloc(ubi_wl_entry_slab, GFP_KERNEL); if (!e) @@ -1768,7 +1771,7 @@ static int erase_aeb(struct ubi_device *ubi, struct ubi_ainf_peb *aeb, bool sync ubi->lookuptbl[e->pnum] = e; if (sync) { - err = ubi_sync_erase(ubi, e, false); + err = ubi_sync_erase(ubi, e, &torture); if (err) goto out_free; diff --git a/drivers/net/amt.c b/drivers/net/amt.c index 182a41d59a75d8..7645ce3a29ad96 100644 --- a/drivers/net/amt.c +++ b/drivers/net/amt.c @@ -3221,6 +3221,9 @@ static int amt_newlink(struct net_device *dev, struct nlattr **tb = params->tb; int err = -EINVAL; + if (!net_eq(link_net, dev_net(dev))) + return err; + amt->net = link_net; amt->mode = nla_get_u32(data[IFLA_AMT_MODE]); diff --git a/drivers/net/bonding/bond_netlink.c b/drivers/net/bonding/bond_netlink.c index 4a11572f663d31..87d92d3cce4a88 100644 --- a/drivers/net/bonding/bond_netlink.c +++ b/drivers/net/bonding/bond_netlink.c @@ -220,7 +220,7 @@ static int bond_changelink(struct net_device *bond_dev, struct nlattr *tb[], struct bonding *bond = netdev_priv(bond_dev); struct bond_opt_value newval; int miimon = 0; - int err; + int err = 0; if (!data) return 0; diff --git a/drivers/net/can/m_can/m_can.c b/drivers/net/can/m_can/m_can.c index eb856547ae7df2..16f80607e150e2 100644 --- a/drivers/net/can/m_can/m_can.c +++ b/drivers/net/can/m_can/m_can.c @@ -2464,7 +2464,7 @@ struct m_can_classdev *m_can_class_allocate_dev(struct device *dev, return ERR_PTR(ret); } - if (dev->of_node && of_property_read_bool(dev->of_node, "wakeup-source")) + if (dev->of_node && of_property_present(dev->of_node, "wakeup-source")) device_set_wakeup_capable(dev, true); /* Get TX FIFO size diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c index 3f5b9592794dac..0880310c9ce3f0 100644 --- a/drivers/net/dsa/b53/b53_common.c +++ b/drivers/net/dsa/b53/b53_common.c @@ -2219,7 +2219,7 @@ int b53_fdb_dump(struct dsa_switch *ds, int port, mutex_unlock(&priv->arl_mutex); - return 0; + return ret; } EXPORT_SYMBOL(b53_fdb_dump); diff --git a/drivers/net/dsa/mv88e6xxx/pcs-6352.c b/drivers/net/dsa/mv88e6xxx/pcs-6352.c index 4228ae5bb9db0f..437054711a2d8b 100644 --- a/drivers/net/dsa/mv88e6xxx/pcs-6352.c +++ b/drivers/net/dsa/mv88e6xxx/pcs-6352.c @@ -305,13 +305,16 @@ static bool mv88e6352_pcs_link_check(struct marvell_c22_pcs *mpcs) struct mv88e6xxx_port *port = mpcs->port; struct mv88e6xxx_chip *chip = port->chip; u8 cmode; + int err; /* Port 4 can be in auto-media mode. Check that the port is * associated with the mpcs. */ mv88e6xxx_reg_lock(chip); - chip->info->ops->port_get_cmode(chip, port->port, &cmode); + err = chip->info->ops->port_get_cmode(chip, port->port, &cmode); mv88e6xxx_reg_unlock(chip); + if (err) + return false; return cmode == MV88E6XXX_PORT_STS_CMODE_100BASEX || cmode == MV88E6XXX_PORT_STS_CMODE_1000BASEX || diff --git a/drivers/net/dsa/mxl862xx/mxl862xx.c b/drivers/net/dsa/mxl862xx/mxl862xx.c index 45d237b3a40f10..cfa7e3e269a28e 100644 --- a/drivers/net/dsa/mxl862xx/mxl862xx.c +++ b/drivers/net/dsa/mxl862xx/mxl862xx.c @@ -2111,6 +2111,7 @@ static int mxl862xx_probe(struct mdio_device *mdiodev) ds->ops = &mxl862xx_switch_ops; ds->phylink_mac_ops = &mxl862xx_phylink_mac_ops; ds->num_ports = MXL862XX_MAX_PORTS; + ds->assisted_learning_on_cpu_port = true; ds->fdb_isolation = true; ds->max_num_bridges = MXL862XX_MAX_BRIDGES; diff --git a/drivers/net/ethernet/airoha/airoha_eth.c b/drivers/net/ethernet/airoha/airoha_eth.c index 79418e682f71f3..2fc8c91f210c64 100644 --- a/drivers/net/ethernet/airoha/airoha_eth.c +++ b/drivers/net/ethernet/airoha/airoha_eth.c @@ -2521,16 +2521,20 @@ static int airoha_qdma_get_tx_ets_stats(struct net_device *netdev, int channel, { struct airoha_gdm_dev *dev = netdev_priv(netdev); struct airoha_qdma *qdma = dev->qdma; + u32 cpu_tx_packets, fwd_tx_packets; + u64 tx_packets; - u64 cpu_tx_packets = airoha_qdma_rr(qdma, REG_CNTR_VAL(channel << 1)); - u64 fwd_tx_packets = airoha_qdma_rr(qdma, - REG_CNTR_VAL((channel << 1) + 1)); - u64 tx_packets = (cpu_tx_packets - dev->cpu_tx_packets) + - (fwd_tx_packets - dev->fwd_tx_packets); + cpu_tx_packets = airoha_qdma_rr(qdma, REG_CNTR_VAL(channel << 1)); + fwd_tx_packets = airoha_qdma_rr(qdma, + REG_CNTR_VAL((channel << 1) + 1)); + tx_packets = (u32)(cpu_tx_packets - + dev->qos_stats[channel].cpu_tx_packets); + tx_packets += (u32)(fwd_tx_packets - + dev->qos_stats[channel].fwd_tx_packets); _bstats_update(opt->stats.bstats, 0, tx_packets); - dev->cpu_tx_packets = cpu_tx_packets; - dev->fwd_tx_packets = fwd_tx_packets; + dev->qos_stats[channel].cpu_tx_packets = cpu_tx_packets; + dev->qos_stats[channel].fwd_tx_packets = fwd_tx_packets; return 0; } diff --git a/drivers/net/ethernet/airoha/airoha_eth.h b/drivers/net/ethernet/airoha/airoha_eth.h index fe934f9ffe8a11..b894828b13752e 100644 --- a/drivers/net/ethernet/airoha/airoha_eth.h +++ b/drivers/net/ethernet/airoha/airoha_eth.h @@ -580,9 +580,10 @@ struct airoha_gdm_dev { struct airoha_eth *eth; DECLARE_BITMAP(qos_sq_bmap, AIROHA_NUM_QOS_CHANNELS); - /* qos stats counters */ - u64 cpu_tx_packets; - u64 fwd_tx_packets; + struct { + u32 cpu_tx_packets; + u32 fwd_tx_packets; + } qos_stats[AIROHA_NUM_QOS_CHANNELS]; u32 flags; int nbq; diff --git a/drivers/net/ethernet/airoha/airoha_npu.c b/drivers/net/ethernet/airoha/airoha_npu.c index b679bed952de37..4045d1eb93ea8b 100644 --- a/drivers/net/ethernet/airoha/airoha_npu.c +++ b/drivers/net/ethernet/airoha/airoha_npu.c @@ -766,7 +766,7 @@ static int airoha_npu_probe(struct platform_device *pdev) npu->irqs[i] = irq; } - err = dma_set_coherent_mask(dev, DMA_BIT_MASK(32)); + err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); if (err) return err; diff --git a/drivers/net/ethernet/allwinner/sun4i-emac.c b/drivers/net/ethernet/allwinner/sun4i-emac.c index fc7341a5cbb7d2..a297874f4a819e 100644 --- a/drivers/net/ethernet/allwinner/sun4i-emac.c +++ b/drivers/net/ethernet/allwinner/sun4i-emac.c @@ -996,9 +996,9 @@ static int emac_probe(struct platform_device *pdev) /* fill in parameters for net-dev structure */ ndev->base_addr = (unsigned long)db->membase; ndev->irq = irq_of_parse_and_map(np, 0); - if (ndev->irq == -ENXIO) { + if (!ndev->irq) { netdev_err(ndev, "No irq resource\n"); - ret = ndev->irq; + ret = -ENXIO; goto out_iounmap; } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 54bae193d1f637..fd2ee24f3b0bcf 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -11923,9 +11923,11 @@ static int bnxt_request_irq(struct bnxt *bp) #endif /* Enable TPH support as part of IRQ request */ - rc = pcie_enable_tph(bp->pdev, PCI_TPH_ST_IV_MODE); - if (!rc) - bp->tph_mode = PCI_TPH_ST_IV_MODE; + if (BNXT_SUPPORTS_QUEUE_API(bp)) { + rc = pcie_enable_tph(bp->pdev, PCI_TPH_ST_IV_MODE); + if (!rc) + bp->tph_mode = PCI_TPH_ST_IV_MODE; + } for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { int map_idx = bnxt_cp_num_to_irq_num(bp, i); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_hwmon.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_hwmon.c index de3427c6c6aaf7..faed0cf726e1e6 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_hwmon.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_hwmon.c @@ -40,7 +40,7 @@ void bnxt_hwmon_notify_event(struct bnxt *bp) return; } - hwmon_notify_event(&bp->pdev->dev, hwmon_temp, attr, 0); + hwmon_notify_event(bp->hwmon_dev, hwmon_temp, attr, 0); } static int bnxt_hwrm_temp_query(struct bnxt *bp, u8 *temp) diff --git a/drivers/net/ethernet/freescale/enetc/enetc.c b/drivers/net/ethernet/freescale/enetc/enetc.c index 8e3f345dd9aacb..80f0082f6c6337 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.c +++ b/drivers/net/ethernet/freescale/enetc/enetc.c @@ -81,6 +81,33 @@ void enetc_reset_mac_addr_filter(struct enetc_mac_filter *filter) } EXPORT_SYMBOL_GPL(enetc_reset_mac_addr_filter); +void enetc_set_congestion_mode(struct enetc_ndev_priv *priv, bool enable) +{ + struct enetc_si *si = priv->si; + struct enetc_hw *hw = &si->hw; + + spin_lock(&si->gen_lock); + + if (enable) + set_bit(ENETC_RXBDR_CM, &priv->flags); + else + clear_bit(ENETC_RXBDR_CM, &priv->flags); + + for (int i = 0; i < priv->num_rx_rings; i++) { + u32 old_rbmr = enetc_rxbdr_rd(hw, i, ENETC_RBMR); + u32 rbmr; + + rbmr = u32_replace_bits(old_rbmr, enable, ENETC_RBMR_CM); + if (rbmr == old_rbmr) + continue; + + enetc_rxbdr_wr(hw, i, ENETC_RBMR, rbmr); + } + + spin_unlock(&si->gen_lock); +} +EXPORT_SYMBOL_GPL(enetc_set_congestion_mode); + static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv) { int num_tx_rings = priv->num_tx_rings; @@ -2632,7 +2659,6 @@ static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring, bool extended) { int idx = rx_ring->index; - u32 rbmr = 0; enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, lower_32_bits(rx_ring->bd_dma_base)); @@ -2660,12 +2686,6 @@ static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring, enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); rx_ring->ext_en = extended; - if (rx_ring->ext_en) - rbmr |= ENETC_RBMR_BDS; - - if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) - rbmr |= ENETC_RBMR_VTE; - rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); rx_ring->idr = hw->reg + ENETC_SIRXIDR; @@ -2676,8 +2696,6 @@ static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring, enetc_lock_mdio(); enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); enetc_unlock_mdio(); - - enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); } static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended) @@ -2704,21 +2722,34 @@ static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) { + struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); int idx = rx_ring->index; - u32 rbmr; + u32 rbmr = ENETC_RBMR_EN; + + if (rx_ring->ext_en) + rbmr |= ENETC_RBMR_BDS; + + if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) + rbmr |= ENETC_RBMR_VTE; + + if (test_bit(ENETC_RXBDR_CM, &priv->flags)) + rbmr |= ENETC_RBMR_CM; - rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR); - rbmr |= ENETC_RBMR_EN; enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); } static void enetc_enable_rx_bdrs(struct enetc_ndev_priv *priv) { - struct enetc_hw *hw = &priv->si->hw; + struct enetc_si *si = priv->si; + struct enetc_hw *hw = &si->hw; int i; + spin_lock(&si->gen_lock); + for (i = 0; i < priv->num_rx_rings; i++) enetc_enable_rxbdr(hw, priv->rx_ring[i]); + + spin_unlock(&si->gen_lock); } static void enetc_enable_tx_bdrs(struct enetc_ndev_priv *priv) @@ -2748,11 +2779,16 @@ static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) static void enetc_disable_rx_bdrs(struct enetc_ndev_priv *priv) { - struct enetc_hw *hw = &priv->si->hw; + struct enetc_si *si = priv->si; + struct enetc_hw *hw = &si->hw; int i; + spin_lock(&si->gen_lock); + for (i = 0; i < priv->num_rx_rings; i++) enetc_disable_rxbdr(hw, priv->rx_ring[i]); + + spin_unlock(&si->gen_lock); } static void enetc_disable_tx_bdrs(struct enetc_ndev_priv *priv) @@ -3344,11 +3380,16 @@ EXPORT_SYMBOL_GPL(enetc_get_stats); static void enetc_enable_rxvlan(struct net_device *ndev, bool en) { struct enetc_ndev_priv *priv = netdev_priv(ndev); - struct enetc_hw *hw = &priv->si->hw; + struct enetc_si *si = priv->si; + struct enetc_hw *hw = &si->hw; int i; + spin_lock(&si->gen_lock); + for (i = 0; i < priv->num_rx_rings; i++) enetc_bdr_enable_rxvlan(hw, i, en); + + spin_unlock(&si->gen_lock); } static void enetc_enable_txvlan(struct net_device *ndev, bool en) @@ -3679,6 +3720,7 @@ int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) si = PTR_ALIGN(p, ENETC_SI_ALIGN); si->pad = (char *)si - (char *)p; + spin_lock_init(&si->gen_lock); pci_set_drvdata(pdev, si); si->pdev = pdev; diff --git a/drivers/net/ethernet/freescale/enetc/enetc.h b/drivers/net/ethernet/freescale/enetc/enetc.h index 04a5dd5ea6c74c..2145563ab2ac13 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.h +++ b/drivers/net/ethernet/freescale/enetc/enetc.h @@ -309,6 +309,13 @@ struct enetc_si { struct net_device *ndev; /* back ref. */ + /* General-purpose lock serializing updates that must not race, + * e.g. read-modify-write of shared hardware registers and of + * selected priv->flags bits between the phylink link callbacks + * and the ring (re)configuration path. + */ + spinlock_t gen_lock; + union { struct enetc_cbdr cbd_ring; /* Only ENETC 1.0 */ struct ntmp_user ntmp_user; /* ENETC 4.1 and later */ @@ -324,8 +331,6 @@ struct enetc_si { const struct enetc_drvdata *drvdata; const struct enetc_si_ops *ops; - struct workqueue_struct *workqueue; - struct work_struct rx_mode_task; struct dentry *debugfs_root; struct enetc_msg_swbd msg; /* Only valid for VSI */ }; @@ -419,6 +424,7 @@ enum enetc_active_offloads { enum enetc_flags_bit { ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS = 0, ENETC_TX_DOWN, + ENETC_RXBDR_CM, }; /* interrupt coalescing modes */ @@ -507,6 +513,7 @@ int enetc_get_driver_data(struct enetc_si *si); void enetc_add_mac_addr_ht_filter(struct enetc_mac_filter *filter, const unsigned char *addr); void enetc_reset_mac_addr_filter(struct enetc_mac_filter *filter); +void enetc_set_congestion_mode(struct enetc_ndev_priv *priv, bool enable); int enetc_open(struct net_device *ndev); int enetc_close(struct net_device *ndev); diff --git a/drivers/net/ethernet/freescale/enetc/enetc4_debugfs.c b/drivers/net/ethernet/freescale/enetc/enetc4_debugfs.c index 1b1591dce73d4b..4a769d9e5679c5 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc4_debugfs.c +++ b/drivers/net/ethernet/freescale/enetc/enetc4_debugfs.c @@ -31,9 +31,11 @@ static int enetc_mac_filter_show(struct seq_file *s, void *data) struct enetc_si *si = s->private; struct enetc_hw *hw = &si->hw; struct maft_entry_data maft; + struct ntmp_user *user; struct enetc_pf *pf; - int i, err, num_si; - u32 val; + u32 val, entry_id; + int i, num_si; + int err = 0; pf = enetc_si_priv(si); num_si = pf->caps.num_vsi + 1; @@ -50,22 +52,30 @@ static int enetc_mac_filter_show(struct seq_file *s, void *data) for (i = 0; i < num_si; i++) enetc_show_si_mac_hash_filter(s, i); - if (!pf->num_mfe) - return 0; + user = &si->ntmp_user; + rtnl_lock(); + + if (bitmap_empty(user->maft_eid_bitmap, user->maft_num_entries)) + goto unlock_rtnl; /* MAC address filter table */ seq_puts(s, "MAC address filter table\n"); - for (i = 0; i < pf->num_mfe; i++) { + for_each_set_bit(entry_id, user->maft_eid_bitmap, + user->maft_num_entries) { memset(&maft, 0, sizeof(maft)); - err = ntmp_maft_query_entry(&si->ntmp_user, i, &maft); + err = ntmp_maft_query_entry(user, entry_id, &maft); if (err) - return err; + goto unlock_rtnl; - seq_printf(s, "Entry %d, MAC: %pM, SI bitmap: 0x%04x\n", i, - maft.keye.mac_addr, le16_to_cpu(maft.cfge.si_bitmap)); + seq_printf(s, "Entry %d, MAC: %pM, SI bitmap: 0x%04x\n", + entry_id, maft.keye.mac_addr, + le16_to_cpu(maft.cfge.si_bitmap)); } - return 0; +unlock_rtnl: + rtnl_unlock(); + + return err; } DEFINE_SHOW_ATTRIBUTE(enetc_mac_filter); diff --git a/drivers/net/ethernet/freescale/enetc/enetc4_hw.h b/drivers/net/ethernet/freescale/enetc/enetc4_hw.h index f18437556a0ee2..56240cd6f0f8b4 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc4_hw.h +++ b/drivers/net/ethernet/freescale/enetc/enetc4_hw.h @@ -69,8 +69,6 @@ /* Port Station interface promiscuous MAC mode register */ #define ENETC4_PSIPMMR 0x200 -#define PSIPMMR_SI_MAC_UP(a) BIT(a) /* a = SI index */ -#define PSIPMMR_SI_MAC_MP(a) BIT((a) + 16) /* Port Station interface promiscuous VLAN mode register */ #define ENETC4_PSIPVMR 0x204 @@ -137,7 +135,6 @@ #define ENETC4_PSIVHFR1(a) ((a) * 0x80 + 0x2064) #define ENETC4_PMCAPR 0x4004 -#define PMCAPR_HD BIT(8) #define PMCAPR_FP GENMASK(10, 9) /* Port capability register */ @@ -200,7 +197,6 @@ #define PM_CMD_CFG_CNT_FRM_EN BIT(13) #define PM_CMD_CFG_TXP BIT(15) #define PM_CMD_CFG_SEND_IDLE BIT(16) -#define PM_CMD_CFG_HD_FCEN BIT(18) #define PM_CMD_CFG_SFD BIT(21) #define PM_CMD_CFG_TX_FLUSH BIT(22) #define PM_CMD_CFG_TX_LOWP_EN BIT(23) diff --git a/drivers/net/ethernet/freescale/enetc/enetc4_pf.c b/drivers/net/ethernet/freescale/enetc/enetc4_pf.c index 437a15bbb47ba8..65d3c24510e872 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc4_pf.c +++ b/drivers/net/ethernet/freescale/enetc/enetc4_pf.c @@ -29,12 +29,6 @@ static void enetc4_get_port_caps(struct enetc_pf *pf) val = enetc_port_rd(hw, ENETC4_ECAPR2); pf->caps.num_rx_bdr = (val & ECAPR2_NUM_RX_BDR) >> 16; pf->caps.num_tx_bdr = val & ECAPR2_NUM_TX_BDR; - - val = enetc_port_rd(hw, ENETC4_PMCAPR); - pf->caps.half_duplex = (val & PMCAPR_HD) ? 1 : 0; - - val = enetc_port_rd(hw, ENETC4_PSIMAFCAPR); - pf->caps.mac_filter_num = val & PSIMAFCAPR_NUM_MAC_AFTE; } static void enetc4_get_psi_hw_features(struct enetc_si *si) @@ -75,38 +69,6 @@ static void enetc4_pf_get_si_primary_mac(struct enetc_hw *hw, int si, put_unaligned_le16(lower, addr + 4); } -static void enetc4_pf_set_si_mac_promisc(struct enetc_hw *hw, int si, - bool uc_promisc, bool mc_promisc) -{ - u32 val = enetc_port_rd(hw, ENETC4_PSIPMMR); - - if (uc_promisc) - val |= PSIPMMR_SI_MAC_UP(si); - else - val &= ~PSIPMMR_SI_MAC_UP(si); - - if (mc_promisc) - val |= PSIPMMR_SI_MAC_MP(si); - else - val &= ~PSIPMMR_SI_MAC_MP(si); - - enetc_port_wr(hw, ENETC4_PSIPMMR, val); -} - -static void enetc4_pf_set_si_uc_hash_filter(struct enetc_hw *hw, int si, - u64 hash) -{ - enetc_port_wr(hw, ENETC4_PSIUMHFR0(si), lower_32_bits(hash)); - enetc_port_wr(hw, ENETC4_PSIUMHFR1(si), upper_32_bits(hash)); -} - -static void enetc4_pf_set_si_mc_hash_filter(struct enetc_hw *hw, int si, - u64 hash) -{ - enetc_port_wr(hw, ENETC4_PSIMMHFR0(si), lower_32_bits(hash)); - enetc_port_wr(hw, ENETC4_PSIMMHFR1(si), upper_32_bits(hash)); -} - static void enetc4_pf_set_loopback(struct net_device *ndev, bool en) { struct enetc_ndev_priv *priv = netdev_priv(ndev); @@ -124,34 +86,47 @@ static void enetc4_pf_set_loopback(struct net_device *ndev, bool en) static void enetc4_pf_clear_maft_entries(struct enetc_pf *pf) { - int i; - - for (i = 0; i < pf->num_mfe; i++) - ntmp_maft_delete_entry(&pf->si->ntmp_user, i); + struct ntmp_user *user = &pf->si->ntmp_user; + u32 entry_id; - pf->num_mfe = 0; + for_each_set_bit(entry_id, user->maft_eid_bitmap, + user->maft_num_entries) { + if (!ntmp_maft_delete_entry(user, entry_id)) + ntmp_clear_eid_bitmap(user->maft_eid_bitmap, entry_id); + } } static int enetc4_pf_add_maft_entries(struct enetc_pf *pf, - struct enetc_mac_addr *mac, - int mac_cnt) + struct netdev_hw_addr_list *uc) { + struct ntmp_user *user = &pf->si->ntmp_user; + int mac_cnt = netdev_hw_addr_list_count(uc); struct maft_entry_data maft = {}; + struct netdev_hw_addr *ha; + u32 available_entries; u16 si_bit = BIT(0); - int i, err; + u32 entry_id; + int err; + + available_entries = user->maft_num_entries - + bitmap_weight(user->maft_eid_bitmap, + user->maft_num_entries); + + if (mac_cnt > available_entries) + return -ENOSPC; maft.cfge.si_bitmap = cpu_to_le16(si_bit); - for (i = 0; i < mac_cnt; i++) { - ether_addr_copy(maft.keye.mac_addr, mac[i].addr); - err = ntmp_maft_add_entry(&pf->si->ntmp_user, i, &maft); + netdev_hw_addr_list_for_each(ha, uc) { + entry_id = ntmp_lookup_free_eid(user->maft_eid_bitmap, + user->maft_num_entries); + ether_addr_copy(maft.keye.mac_addr, ha->addr); + err = ntmp_maft_add_entry(user, entry_id, &maft); if (unlikely(err)) { - pf->num_mfe = i; + ntmp_clear_eid_bitmap(user->maft_eid_bitmap, entry_id); goto clear_maft_entries; } } - pf->num_mfe = mac_cnt; - return 0; clear_maft_entries: @@ -160,97 +135,70 @@ clear_maft_entries: return err; } -static int enetc4_pf_set_uc_exact_filter(struct enetc_pf *pf) +static void enetc4_pf_set_uc_hash_filter(struct enetc_pf *pf, + struct netdev_hw_addr_list *uc) { - int max_num_mfe = pf->caps.mac_filter_num; - struct enetc_mac_filter mac_filter = {}; - struct net_device *ndev = pf->si->ndev; - struct enetc_hw *hw = &pf->si->hw; - struct enetc_mac_addr *mac_tbl; + struct enetc_mac_filter *mac_filter = &pf->mac_filter[UC]; struct netdev_hw_addr *ha; - int i = 0, err; - int mac_cnt; + u64 hash; - netif_addr_lock_bh(ndev); + enetc_reset_mac_addr_filter(mac_filter); + netdev_hw_addr_list_for_each(ha, uc) + enetc_add_mac_addr_ht_filter(mac_filter, ha->addr); - mac_cnt = netdev_uc_count(ndev); - if (!mac_cnt) { - netif_addr_unlock_bh(ndev); + bitmap_to_arr64(&hash, mac_filter->mac_hash_table, + ENETC_MADDR_HASH_TBL_SZ); + enetc_set_si_uc_hash_filter(pf->si, 0, hash); +} + +static int enetc4_pf_set_uc_exact_filter(struct enetc_pf *pf, + struct netdev_hw_addr_list *uc) +{ + struct enetc_si *si = pf->si; + int err; + + if (netdev_hw_addr_list_empty(uc)) { /* clear both MAC hash and exact filters */ - enetc4_pf_set_si_uc_hash_filter(hw, 0, 0); + enetc_set_si_uc_hash_filter(si, 0, 0); enetc4_pf_clear_maft_entries(pf); return 0; } - if (mac_cnt > max_num_mfe) { - err = -ENOSPC; - goto unlock_netif_addr; - } - - mac_tbl = kzalloc_objs(*mac_tbl, mac_cnt, GFP_ATOMIC); - if (!mac_tbl) { - err = -ENOMEM; - goto unlock_netif_addr; - } - - netdev_for_each_uc_addr(ha, ndev) { - enetc_add_mac_addr_ht_filter(&mac_filter, ha->addr); - ether_addr_copy(mac_tbl[i++].addr, ha->addr); - } - - netif_addr_unlock_bh(ndev); - - /* Set temporary unicast hash filters in case of Rx loss when + /* Set temporary unicast hash filter in case of Rx loss when * updating MAC address filter table */ - enetc4_pf_set_si_uc_hash_filter(hw, 0, *mac_filter.mac_hash_table); + enetc4_pf_set_uc_hash_filter(pf, uc); enetc4_pf_clear_maft_entries(pf); - if (!enetc4_pf_add_maft_entries(pf, mac_tbl, i)) - enetc4_pf_set_si_uc_hash_filter(hw, 0, 0); - - kfree(mac_tbl); - - return 0; - -unlock_netif_addr: - netif_addr_unlock_bh(ndev); + err = enetc4_pf_add_maft_entries(pf, uc); + if (!err) { + enetc_reset_mac_addr_filter(&pf->mac_filter[UC]); + enetc_set_si_uc_hash_filter(si, 0, 0); + } return err; } -static void enetc4_pf_set_mac_hash_filter(struct enetc_pf *pf, int type) +static void enetc4_pf_set_mc_hash_filter(struct enetc_pf *pf, + struct netdev_hw_addr_list *mc) { - struct net_device *ndev = pf->si->ndev; - struct enetc_mac_filter *mac_filter; - struct enetc_hw *hw = &pf->si->hw; + struct enetc_mac_filter *mac_filter = &pf->mac_filter[MC]; struct netdev_hw_addr *ha; + u64 hash; - netif_addr_lock_bh(ndev); - if (type & ENETC_MAC_FILTER_TYPE_UC) { - mac_filter = &pf->mac_filter[UC]; - enetc_reset_mac_addr_filter(mac_filter); - netdev_for_each_uc_addr(ha, ndev) - enetc_add_mac_addr_ht_filter(mac_filter, ha->addr); + enetc_reset_mac_addr_filter(mac_filter); + netdev_hw_addr_list_for_each(ha, mc) + enetc_add_mac_addr_ht_filter(mac_filter, ha->addr); - enetc4_pf_set_si_uc_hash_filter(hw, 0, - *mac_filter->mac_hash_table); - } - - if (type & ENETC_MAC_FILTER_TYPE_MC) { - mac_filter = &pf->mac_filter[MC]; - enetc_reset_mac_addr_filter(mac_filter); - netdev_for_each_mc_addr(ha, ndev) - enetc_add_mac_addr_ht_filter(mac_filter, ha->addr); - - enetc4_pf_set_si_mc_hash_filter(hw, 0, - *mac_filter->mac_hash_table); - } - netif_addr_unlock_bh(ndev); + bitmap_to_arr64(&hash, mac_filter->mac_hash_table, + ENETC_MADDR_HASH_TBL_SZ); + enetc_set_si_mc_hash_filter(pf->si, 0, hash); } -static void enetc4_pf_set_mac_filter(struct enetc_pf *pf, int type) +static void enetc4_pf_set_mac_filter(struct enetc_pf *pf, int type, + struct netdev_hw_addr_list *uc, + struct netdev_hw_addr_list *mc) { /* Currently, the MAC address filter table (MAFT) only has 4 entries, * and multiple multicast addresses for filtering will be configured @@ -258,15 +206,16 @@ static void enetc4_pf_set_mac_filter(struct enetc_pf *pf, int type) * unicast filtering. If the number of unicast addresses exceeds the * table capacity, the MAC hash filter will be used. */ - if (type & ENETC_MAC_FILTER_TYPE_UC && enetc4_pf_set_uc_exact_filter(pf)) { + if (type & ENETC_MAC_FILTER_TYPE_UC && + enetc4_pf_set_uc_exact_filter(pf, uc)) { /* Fall back to the MAC hash filter */ - enetc4_pf_set_mac_hash_filter(pf, ENETC_MAC_FILTER_TYPE_UC); + enetc4_pf_set_uc_hash_filter(pf, uc); /* Clear the old MAC exact filter */ enetc4_pf_clear_maft_entries(pf); } if (type & ENETC_MAC_FILTER_TYPE_MC) - enetc4_pf_set_mac_hash_filter(pf, ENETC_MAC_FILTER_TYPE_MC); + enetc4_pf_set_mc_hash_filter(pf, mc); } static const struct enetc_pf_ops enetc4_pf_ops = { @@ -450,18 +399,60 @@ static void enetc4_configure_port(struct enetc_pf *pf) enetc_set_default_rss_key(pf); } +static void enetc4_get_ntmp_caps(struct enetc_si *si) +{ + struct ntmp_user *user = &si->ntmp_user; + struct enetc_hw *hw = &si->hw; + u32 val; + + val = enetc_port_rd(hw, ENETC4_PSIMAFCAPR); + user->maft_num_entries = FIELD_GET(PSIMAFCAPR_NUM_MAC_AFTE, val); +} + +static int enetc4_ntmp_bitmap_init(struct ntmp_user *user) +{ + user->maft_eid_bitmap = bitmap_zalloc(user->maft_num_entries, + GFP_KERNEL); + if (!user->maft_eid_bitmap) + return -ENOMEM; + + return 0; +} + +static void enetc4_ntmp_bitmap_free(struct ntmp_user *user) +{ + bitmap_free(user->maft_eid_bitmap); + user->maft_eid_bitmap = NULL; +} + static int enetc4_init_ntmp_user(struct enetc_si *si) { struct ntmp_user *user = &si->ntmp_user; + int err; /* For ENETC 4.1, all table versions are 0 */ memset(&user->tbl, 0, sizeof(user->tbl)); - return enetc4_setup_cbdr(si); + err = enetc4_setup_cbdr(si); + if (err) + return err; + + enetc4_get_ntmp_caps(si); + err = enetc4_ntmp_bitmap_init(user); + if (err) + goto teardown_cbdr; + + return 0; + +teardown_cbdr: + enetc4_teardown_cbdr(si); + + return err; } static void enetc4_free_ntmp_user(struct enetc_si *si) { + enetc4_ntmp_bitmap_free(&si->ntmp_user); enetc4_teardown_cbdr(si); } @@ -479,7 +470,7 @@ static int enetc4_pf_init(struct enetc_pf *pf) err = enetc4_init_ntmp_user(pf->si); if (err) { - dev_err(dev, "Failed to init CBDR\n"); + dev_err(dev, "Failed to init NTMP user\n"); return err; } @@ -493,18 +484,17 @@ static void enetc4_pf_free(struct enetc_pf *pf) enetc4_free_ntmp_user(pf->si); } -static void enetc4_psi_do_set_rx_mode(struct work_struct *work) +static int enetc4_pf_set_rx_mode(struct net_device *ndev, + struct netdev_hw_addr_list *uc, + struct netdev_hw_addr_list *mc) { - struct enetc_si *si = container_of(work, struct enetc_si, rx_mode_task); - struct enetc_pf *pf = enetc_si_priv(si); - struct net_device *ndev = si->ndev; - struct enetc_hw *hw = &si->hw; + struct enetc_ndev_priv *priv = netdev_priv(ndev); + struct enetc_pf *pf = enetc_si_priv(priv->si); + struct enetc_si *si = priv->si; bool uc_promisc = false; bool mc_promisc = false; int type = 0; - rtnl_lock(); - if (ndev->flags & IFF_PROMISC) { uc_promisc = true; mc_promisc = true; @@ -515,28 +505,21 @@ static void enetc4_psi_do_set_rx_mode(struct work_struct *work) type = ENETC_MAC_FILTER_TYPE_ALL; } - enetc4_pf_set_si_mac_promisc(hw, 0, uc_promisc, mc_promisc); + enetc_set_si_uc_promisc(si, 0, uc_promisc); + enetc_set_si_mc_promisc(si, 0, mc_promisc); if (uc_promisc) { - enetc4_pf_set_si_uc_hash_filter(hw, 0, 0); + enetc_set_si_uc_hash_filter(si, 0, 0); enetc4_pf_clear_maft_entries(pf); } if (mc_promisc) - enetc4_pf_set_si_mc_hash_filter(hw, 0, 0); + enetc_set_si_mc_hash_filter(si, 0, 0); /* Set new MAC filter */ - enetc4_pf_set_mac_filter(pf, type); + enetc4_pf_set_mac_filter(pf, type, uc, mc); - rtnl_unlock(); -} - -static void enetc4_pf_set_rx_mode(struct net_device *ndev) -{ - struct enetc_ndev_priv *priv = netdev_priv(ndev); - struct enetc_si *si = priv->si; - - queue_work(si->workqueue, &si->rx_mode_task); + return 0; } static int enetc4_pf_set_features(struct net_device *ndev, @@ -566,7 +549,7 @@ static const struct net_device_ops enetc4_ndev_ops = { .ndo_start_xmit = enetc_xmit, .ndo_get_stats = enetc_get_stats, .ndo_set_mac_address = enetc_pf_set_mac_addr, - .ndo_set_rx_mode = enetc4_pf_set_rx_mode, + .ndo_set_rx_mode_async = enetc4_pf_set_rx_mode, .ndo_set_features = enetc4_pf_set_features, .ndo_vlan_rx_add_vid = enetc_vlan_rx_add_vid, .ndo_vlan_rx_kill_vid = enetc_vlan_rx_del_vid, @@ -602,11 +585,6 @@ static void enetc4_mac_config(struct enetc_pf *pf, unsigned int mode, case PHY_INTERFACE_MODE_RGMII_RXID: case PHY_INTERFACE_MODE_RGMII_TXID: val |= IFMODE_RGMII; - /* We need to enable auto-negotiation for the MAC - * if its RGMII interface support In-Band status. - */ - if (phylink_autoneg_inband(mode)) - val |= PM_IF_MODE_ENA; break; case PHY_INTERFACE_MODE_RMII: val |= IFMODE_RMII; @@ -717,22 +695,6 @@ static void enetc4_set_rmii_mac(struct enetc_pf *pf, int speed, int duplex) enetc_port_mac_wr(si, ENETC4_PM_IF_MODE(0), val); } -static void enetc4_set_hd_flow_control(struct enetc_pf *pf, bool enable) -{ - struct enetc_si *si = pf->si; - u32 old_val, val; - - if (!pf->caps.half_duplex) - return; - - old_val = enetc_port_mac_rd(si, ENETC4_PM_CMD_CFG(0)); - val = u32_replace_bits(old_val, enable ? 1 : 0, PM_CMD_CFG_HD_FCEN); - if (val == old_val) - return; - - enetc_port_mac_wr(si, ENETC4_PM_CMD_CFG(0), val); -} - static void enetc4_set_rx_pause(struct enetc_pf *pf, bool rx_pause) { struct enetc_si *si = pf->si; @@ -746,22 +708,14 @@ static void enetc4_set_rx_pause(struct enetc_pf *pf, bool rx_pause) enetc_port_mac_wr(si, ENETC4_PM_CMD_CFG(0), val); } -static void enetc4_set_tx_pause(struct enetc_pf *pf, int num_rxbdr, bool tx_pause) +static void enetc4_set_tx_pause(struct enetc_pf *pf, bool tx_pause) { + struct enetc_ndev_priv *priv = netdev_priv(pf->si->ndev); u32 pause_off_thresh = 0, pause_on_thresh = 0; u32 init_quanta = 0, refresh_quanta = 0; struct enetc_hw *hw = &pf->si->hw; - u32 rbmr, old_rbmr; - int i; - - for (i = 0; i < num_rxbdr; i++) { - old_rbmr = enetc_rxbdr_rd(hw, i, ENETC_RBMR); - rbmr = u32_replace_bits(old_rbmr, tx_pause ? 1 : 0, ENETC_RBMR_CM); - if (rbmr == old_rbmr) - continue; - enetc_rxbdr_wr(hw, i, ENETC_RBMR, rbmr); - } + enetc_set_congestion_mode(priv, tx_pause); if (tx_pause) { /* When the port first enters congestion, send a PAUSE request @@ -908,13 +862,11 @@ static void enetc4_pl_mac_link_up(struct phylink_config *config, struct enetc_pf *pf = phylink_to_enetc_pf(config); struct enetc_si *si = pf->si; struct enetc_ndev_priv *priv; - bool hd_fc = false; priv = netdev_priv(si->ndev); enetc4_set_port_speed(priv, speed); - if (!phylink_autoneg_inband(mode) && - phy_interface_mode_is_rgmii(interface)) + if (phy_interface_mode_is_rgmii(interface)) enetc4_set_rgmii_mac(pf, speed, duplex); if (interface == PHY_INTERFACE_MODE_RMII) @@ -926,19 +878,9 @@ static void enetc4_pl_mac_link_up(struct phylink_config *config, */ if (priv->active_offloads & ENETC_F_QBU) tx_pause = false; - } else { /* DUPLEX_HALF */ - if (tx_pause || rx_pause) - hd_fc = true; - - /* As per 802.3 annex 31B, PAUSE frames are only supported - * when the link is configured for full duplex operation. - */ - tx_pause = false; - rx_pause = false; } - enetc4_set_hd_flow_control(pf, hd_fc); - enetc4_set_tx_pause(pf, priv->num_rx_rings, tx_pause); + enetc4_set_tx_pause(pf, tx_pause); enetc4_set_rx_pause(pf, rx_pause); enetc4_mac_tx_enable(pf); enetc4_mac_rx_enable(pf); @@ -1009,19 +951,6 @@ static void enetc4_link_deinit(struct enetc_ndev_priv *priv) enetc_mdiobus_destroy(pf); } -static int enetc4_psi_wq_task_init(struct enetc_si *si) -{ - char wq_name[24]; - - INIT_WORK(&si->rx_mode_task, enetc4_psi_do_set_rx_mode); - snprintf(wq_name, sizeof(wq_name), "enetc-%s", pci_name(si->pdev)); - si->workqueue = create_singlethread_workqueue(wq_name); - if (!si->workqueue) - return -ENOMEM; - - return 0; -} - static int enetc4_pf_netdev_create(struct enetc_si *si) { struct device *dev = &si->pdev->dev; @@ -1062,12 +991,6 @@ static int enetc4_pf_netdev_create(struct enetc_si *si) if (err) goto err_link_init; - err = enetc4_psi_wq_task_init(si); - if (err) { - dev_err(dev, "Failed to init workqueue\n"); - goto err_wq_init; - } - err = register_netdev(ndev); if (err) { dev_err(dev, "Failed to register netdev\n"); @@ -1077,8 +1000,6 @@ static int enetc4_pf_netdev_create(struct enetc_si *si) return 0; err_reg_netdev: - destroy_workqueue(si->workqueue); -err_wq_init: enetc4_link_deinit(priv); err_link_init: enetc_free_msix(priv); @@ -1096,8 +1017,6 @@ static void enetc4_pf_netdev_destroy(struct enetc_si *si) struct net_device *ndev = si->ndev; unregister_netdev(ndev); - cancel_work(&si->rx_mode_task); - destroy_workqueue(si->workqueue); enetc4_link_deinit(priv); enetc_free_msix(priv); free_netdev(ndev); diff --git a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c index 71f376ef1be1c9..07b7832f24276a 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c @@ -29,7 +29,7 @@ static const u32 enetc_rxbdr_regs[] = { }; static const u32 enetc_port_regs[] = { - ENETC_PMR, ENETC_PSR, ENETC_PSIPMR, ENETC_PSIPMAR0(0), + ENETC_PMR, ENETC_PSR, ENETC_PSIPMMR, ENETC_PSIPMAR0(0), ENETC_PSIPMAR1(0), ENETC_PTXMBAR, ENETC_PCAPR0, ENETC_PCAPR1, ENETC_PSICFGR0(0), ENETC_PRFSCAPR, ENETC_PTCMSDUR(0), ENETC_PM0_CMD_CFG, ENETC_PM0_MAXFRM, ENETC_PM0_IF_MODE diff --git a/drivers/net/ethernet/freescale/enetc/enetc_hw.h b/drivers/net/ethernet/freescale/enetc/enetc_hw.h index bf99b65d7598ab..66bfda60da9cbb 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_hw.h +++ b/drivers/net/ethernet/freescale/enetc/enetc_hw.h @@ -180,9 +180,10 @@ enum enetc_bdr_type {TX, RX}; #define ENETC_PMR_PSPEED_1000M BIT(9) #define ENETC_PMR_PSPEED_2500M BIT(10) #define ENETC_PSR 0x0004 /* RO */ -#define ENETC_PSIPMR 0x0018 -#define ENETC_PSIPMR_SET_UP(n) BIT(n) /* n = SI index */ -#define ENETC_PSIPMR_SET_MP(n) BIT((n) + 16) +#define ENETC_PSIPMMR 0x0018 +#define PSIPMMR_SI_MAC_UP(n) BIT(n) /* n = SI index */ +#define PSIPMMR_SI_MAC_MP(n) BIT((n) + 16) + #define ENETC_PSIPVMR 0x001c #define ENETC_VLAN_PROMISC_MAP_ALL 0x7 #define ENETC_PSIPVMR_SET_VP(simap) ((simap) & 0x7) diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf.c b/drivers/net/ethernet/freescale/enetc/enetc_pf.c index 2d687bb8c3a063..bdfe6bc056b52a 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_pf.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.c @@ -80,37 +80,6 @@ static void enetc_add_mac_addr_em_filter(struct enetc_mac_filter *filter, filter->mac_addr_cnt++; } -static void enetc_clear_mac_ht_flt(struct enetc_si *si, int si_idx, int type) -{ - bool err = si->errata & ENETC_ERR_UCMCSWP; - - if (type == UC) { - enetc_port_wr(&si->hw, ENETC_PSIUMHFR0(si_idx, err), 0); - enetc_port_wr(&si->hw, ENETC_PSIUMHFR1(si_idx), 0); - } else { /* MC */ - enetc_port_wr(&si->hw, ENETC_PSIMMHFR0(si_idx, err), 0); - enetc_port_wr(&si->hw, ENETC_PSIMMHFR1(si_idx), 0); - } -} - -static void enetc_set_mac_ht_flt(struct enetc_si *si, int si_idx, int type, - unsigned long hash) -{ - bool err = si->errata & ENETC_ERR_UCMCSWP; - - if (type == UC) { - enetc_port_wr(&si->hw, ENETC_PSIUMHFR0(si_idx, err), - lower_32_bits(hash)); - enetc_port_wr(&si->hw, ENETC_PSIUMHFR1(si_idx), - upper_32_bits(hash)); - } else { /* MC */ - enetc_port_wr(&si->hw, ENETC_PSIMMHFR0(si_idx, err), - lower_32_bits(hash)); - enetc_port_wr(&si->hw, ENETC_PSIMMHFR1(si_idx), - upper_32_bits(hash)); - } -} - static void enetc_sync_mac_filters(struct enetc_pf *pf) { struct enetc_mac_filter *f = pf->mac_filter; @@ -122,12 +91,16 @@ static void enetc_sync_mac_filters(struct enetc_pf *pf) for (i = 0; i < MADDR_TYPE; i++, f++) { bool em = (f->mac_addr_cnt == 1) && (i == UC); bool clear = !f->mac_addr_cnt; + u64 hash; if (clear) { - if (i == UC) + if (i == UC) { enetc_clear_mac_flt_entry(si, pos); + enetc_set_si_uc_hash_filter(si, 0, 0); + } else { + enetc_set_si_mc_hash_filter(si, 0, 0); + } - enetc_clear_mac_ht_flt(si, 0, i); continue; } @@ -135,7 +108,7 @@ static void enetc_sync_mac_filters(struct enetc_pf *pf) if (em) { int err; - enetc_clear_mac_ht_flt(si, 0, UC); + enetc_set_si_uc_hash_filter(si, 0, 0); err = enetc_set_mac_flt_entry(si, pos, f->mac_addr, BIT(0)); @@ -147,11 +120,15 @@ static void enetc_sync_mac_filters(struct enetc_pf *pf) err); } + bitmap_to_arr64(&hash, f->mac_hash_table, + ENETC_MADDR_HASH_TBL_SZ); /* hash table filter, clear EM filter for UC entries */ - if (i == UC) + if (i == UC) { enetc_clear_mac_flt_entry(si, pos); - - enetc_set_mac_ht_flt(si, 0, i, *f->mac_hash_table); + enetc_set_si_uc_hash_filter(si, 0, hash); + } else { + enetc_set_si_mc_hash_filter(si, 0, hash); + } } } @@ -159,21 +136,17 @@ static void enetc_pf_set_rx_mode(struct net_device *ndev) { struct enetc_ndev_priv *priv = netdev_priv(ndev); struct enetc_pf *pf = enetc_si_priv(priv->si); - struct enetc_hw *hw = &priv->si->hw; bool uprom = false, mprom = false; struct enetc_mac_filter *filter; struct netdev_hw_addr *ha; - u32 psipmr = 0; bool em; if (ndev->flags & IFF_PROMISC) { /* enable promisc mode for SI0 (PF) */ - psipmr = ENETC_PSIPMR_SET_UP(0) | ENETC_PSIPMR_SET_MP(0); uprom = true; mprom = true; } else if (ndev->flags & IFF_ALLMULTI) { /* enable multi cast promisc mode for SI0 (PF) */ - psipmr = ENETC_PSIPMR_SET_MP(0); mprom = true; } @@ -211,9 +184,8 @@ static void enetc_pf_set_rx_mode(struct net_device *ndev) /* update PF entries */ enetc_sync_mac_filters(pf); - psipmr |= enetc_port_rd(hw, ENETC_PSIPMR) & - ~(ENETC_PSIPMR_SET_UP(0) | ENETC_PSIPMR_SET_MP(0)); - enetc_port_wr(hw, ENETC_PSIPMR, psipmr); + enetc_set_si_uc_promisc(priv->si, 0, uprom); + enetc_set_si_mc_promisc(priv->si, 0, mprom); } static void enetc_set_loopback(struct net_device *ndev, bool en) @@ -474,7 +446,7 @@ static void enetc_configure_port(struct enetc_pf *pf) pf->vlan_promisc_simap = ENETC_VLAN_PROMISC_MAP_ALL; enetc_set_vlan_promisc(hw, pf->vlan_promisc_simap); - enetc_port_wr(hw, ENETC_PSIPMR, 0); + enetc_port_wr(hw, ENETC_PSIPMMR, 0); /* enable port */ enetc_port_wr(hw, ENETC_PMR, ENETC_PMR_EN); @@ -608,8 +580,7 @@ static void enetc_pl_mac_link_up(struct phylink_config *config, struct enetc_hw *hw = &pf->si->hw; struct enetc_si *si = pf->si; struct enetc_ndev_priv *priv; - u32 rbmr, cmd_cfg; - int idx; + u32 cmd_cfg; priv = netdev_priv(pf->si->ndev); @@ -621,16 +592,7 @@ static void enetc_pl_mac_link_up(struct phylink_config *config, enetc_force_rgmii_mac(si, speed, duplex); /* Flow control */ - for (idx = 0; idx < priv->num_rx_rings; idx++) { - rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR); - - if (tx_pause) - rbmr |= ENETC_RBMR_CM; - else - rbmr &= ~ENETC_RBMR_CM; - - enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); - } + enetc_set_congestion_mode(priv, tx_pause); if (tx_pause) { /* When the port first enters congestion, send a PAUSE request diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf.h b/drivers/net/ethernet/freescale/enetc/enetc_pf.h index 285b7e5c48fdc7..7e886dc49997f1 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_pf.h +++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.h @@ -17,12 +17,10 @@ struct enetc_vf_state { }; struct enetc_port_caps { - u32 half_duplex:1; int num_vsi; int num_msix; int num_rx_bdr; int num_tx_bdr; - int mac_filter_num; }; struct enetc_pf; @@ -60,8 +58,6 @@ struct enetc_pf { struct enetc_port_caps caps; const struct enetc_pf_ops *ops; - - int num_mfe; /* number of mac address filter table entries */ }; #define phylink_to_enetc_pf(config) \ diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf_common.c b/drivers/net/ethernet/freescale/enetc/enetc_pf_common.c index 6e5d2f8699150b..3597cb81a7ccbc 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_pf_common.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_pf_common.c @@ -87,6 +87,90 @@ int enetc_setup_mac_addresses(struct device_node *np, struct enetc_pf *pf) } EXPORT_SYMBOL_GPL(enetc_setup_mac_addresses); +void enetc_set_si_uc_promisc(struct enetc_si *si, int si_id, bool promisc) +{ + struct enetc_hw *hw = &si->hw; + int psipmmr_off; + u32 val; + + if (is_enetc_rev1(si)) + psipmmr_off = ENETC_PSIPMMR; + else + psipmmr_off = ENETC4_PSIPMMR; + + val = enetc_port_rd(hw, psipmmr_off); + + if (promisc) + val |= PSIPMMR_SI_MAC_UP(si_id); + else + val &= ~PSIPMMR_SI_MAC_UP(si_id); + + enetc_port_wr(hw, psipmmr_off, val); +} +EXPORT_SYMBOL_GPL(enetc_set_si_uc_promisc); + +void enetc_set_si_mc_promisc(struct enetc_si *si, int si_id, bool promisc) +{ + struct enetc_hw *hw = &si->hw; + int psipmmr_off; + u32 val; + + if (is_enetc_rev1(si)) + psipmmr_off = ENETC_PSIPMMR; + else + psipmmr_off = ENETC4_PSIPMMR; + + val = enetc_port_rd(hw, psipmmr_off); + + if (promisc) + val |= PSIPMMR_SI_MAC_MP(si_id); + else + val &= ~PSIPMMR_SI_MAC_MP(si_id); + + enetc_port_wr(hw, psipmmr_off, val); +} +EXPORT_SYMBOL_GPL(enetc_set_si_mc_promisc); + +void enetc_set_si_uc_hash_filter(struct enetc_si *si, int si_id, u64 hash) +{ + int psiumhfr0_off, psiumhfr1_off; + struct enetc_hw *hw = &si->hw; + + if (is_enetc_rev1(si)) { + bool err = si->errata & ENETC_ERR_UCMCSWP; + + psiumhfr0_off = ENETC_PSIUMHFR0(si_id, err); + psiumhfr1_off = ENETC_PSIUMHFR1(si_id); + } else { + psiumhfr0_off = ENETC4_PSIUMHFR0(si_id); + psiumhfr1_off = ENETC4_PSIUMHFR1(si_id); + } + + enetc_port_wr(hw, psiumhfr0_off, lower_32_bits(hash)); + enetc_port_wr(hw, psiumhfr1_off, upper_32_bits(hash)); +} +EXPORT_SYMBOL_GPL(enetc_set_si_uc_hash_filter); + +void enetc_set_si_mc_hash_filter(struct enetc_si *si, int si_id, u64 hash) +{ + int psimmhfr0_off, psimmhfr1_off; + struct enetc_hw *hw = &si->hw; + + if (is_enetc_rev1(si)) { + bool err = si->errata & ENETC_ERR_UCMCSWP; + + psimmhfr0_off = ENETC_PSIMMHFR0(si_id, err); + psimmhfr1_off = ENETC_PSIMMHFR1(si_id); + } else { + psimmhfr0_off = ENETC4_PSIMMHFR0(si_id); + psimmhfr1_off = ENETC4_PSIMMHFR1(si_id); + } + + enetc_port_wr(hw, psimmhfr0_off, lower_32_bits(hash)); + enetc_port_wr(hw, psimmhfr1_off, upper_32_bits(hash)); +} +EXPORT_SYMBOL_GPL(enetc_set_si_mc_hash_filter); + void enetc_pf_netdev_setup(struct enetc_si *si, struct net_device *ndev, const struct net_device_ops *ndev_ops) { diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf_common.h b/drivers/net/ethernet/freescale/enetc/enetc_pf_common.h index 57d2e0ebd2b0bb..bf9029b0a0173d 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_pf_common.h +++ b/drivers/net/ethernet/freescale/enetc/enetc_pf_common.h @@ -17,6 +17,10 @@ void enetc_set_default_rss_key(struct enetc_pf *pf); int enetc_vlan_rx_add_vid(struct net_device *ndev, __be16 prot, u16 vid); int enetc_vlan_rx_del_vid(struct net_device *ndev, __be16 prot, u16 vid); int enetc_init_sriov_resources(struct enetc_pf *pf); +void enetc_set_si_uc_promisc(struct enetc_si *si, int si_id, bool promisc); +void enetc_set_si_mc_promisc(struct enetc_si *si, int si_id, bool promisc); +void enetc_set_si_uc_hash_filter(struct enetc_si *si, int si_id, u64 hash); +void enetc_set_si_mc_hash_filter(struct enetc_si *si, int si_id, u64 hash); static inline u16 enetc_get_ip_revision(struct enetc_hw *hw) { diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index ced4dbf8cd90fd..794ec427b0ee64 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -5457,7 +5457,8 @@ failed_mii_init: failed_irq: fec_enet_deinit(ndev); failed_init: - fec_ptp_stop(pdev); + if (fep->bufdesc_ex) + fec_ptp_stop(pdev); failed_reset: pm_runtime_put_noidle(&pdev->dev); pm_runtime_disable(&pdev->dev); @@ -5499,7 +5500,8 @@ fec_drv_remove(struct platform_device *pdev) ERR_PTR(ret)); cancel_work_sync(&fep->tx_timeout_work); - fec_ptp_stop(pdev); + if (fep->bufdesc_ex) + fec_ptp_stop(pdev); unregister_netdev(ndev); fec_enet_mii_remove(fep); if (fep->reg_phy) diff --git a/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c b/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c index 0ae31499467693..3e6ae2aa0d4b48 100644 --- a/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c +++ b/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c @@ -553,7 +553,7 @@ static int hbg_ring_page_pool_init(struct hbg_priv *priv, struct hbg_ring *ring) .nid = dev_to_node(&priv->pdev->dev), .dev = &priv->pdev->dev, .napi = &ring->napi, - .dma_dir = DMA_FROM_DEVICE, + .dma_dir = DMA_BIDIRECTIONAL, .offset = 0, .max_len = hbg_get_page_size(ring), }; diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_tx.c b/drivers/net/ethernet/huawei/hinic3/hinic3_tx.c index 9306bf0020cafe..cc541e7a231869 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_tx.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_tx.c @@ -261,8 +261,7 @@ static int hinic3_tx_csum(struct hinic3_txq *txq, struct hinic3_sq_task *task, ((struct udphdr *)skb_transport_header(skb))->dest != VXLAN_OFFLOAD_PORT_LE) { /* Unsupported tunnel packet, disable csum offload */ - skb_checksum_help(skb); - return 0; + return skb_checksum_help(skb); } } @@ -412,6 +411,10 @@ static u32 hinic3_tx_offload(struct sk_buff *skb, struct hinic3_sq_task *task, offload |= HINIC3_TX_OFFLOAD_TSO; } else { tso_cs_en = hinic3_tx_csum(txq, task, skb); + if (tso_cs_en < 0) { + offload = HINIC3_TX_OFFLOAD_INVALID; + return offload; + } if (tso_cs_en) offload |= HINIC3_TX_OFFLOAD_CSUM; } @@ -545,6 +548,7 @@ static netdev_tx_t hinic3_send_one_skb(struct sk_buff *skb, skb->len = MIN_SKB_LEN; } + offload = hinic3_tx_offload(skb, &task, &queue_info, txq); num_sge = skb_shinfo(skb)->nr_frags + 1; /* assume normal wqe format + 1 wqebb for task info */ wqebb_cnt = num_sge + 1; @@ -560,7 +564,6 @@ static netdev_tx_t hinic3_send_one_skb(struct sk_buff *skb, return NETDEV_TX_BUSY; } - offload = hinic3_tx_offload(skb, &task, &queue_info, txq); if (unlikely(offload == HINIC3_TX_OFFLOAD_INVALID)) { goto err_drop_pkt; } else if (!offload) { diff --git a/drivers/net/ethernet/intel/ice/devlink/devlink.c b/drivers/net/ethernet/intel/ice/devlink/devlink.c index 22b7d8e6bd9e8f..8c2b63eef82bd5 100644 --- a/drivers/net/ethernet/intel/ice/devlink/devlink.c +++ b/drivers/net/ethernet/intel/ice/devlink/devlink.c @@ -1890,27 +1890,18 @@ static int ice_devlink_nvm_snapshot(struct devlink *devlink, */ for (i = 0; i < num_blks; i++) { u32 read_sz = min_t(u32, ICE_DEVLINK_READ_BLK_SIZE, left); - - status = ice_acquire_nvm(hw, ICE_RES_READ); - if (status) { - dev_dbg(dev, "ice_acquire_nvm failed, err %d aq_err %d\n", - status, hw->adminq.sq_last_status); - NL_SET_ERR_MSG_MOD(extack, "Failed to acquire NVM semaphore"); - vfree(nvm_data); - return -EIO; - } + enum libie_aq_err read_aq_err = LIBIE_AQ_RC_OK; status = ice_read_flat_nvm(hw, i * ICE_DEVLINK_READ_BLK_SIZE, - &read_sz, tmp, read_shadow_ram); + &read_sz, tmp, read_shadow_ram, + &read_aq_err); if (status) { dev_dbg(dev, "ice_read_flat_nvm failed after reading %u bytes, err %d aq_err %d\n", - read_sz, status, hw->adminq.sq_last_status); + read_sz, status, read_aq_err); NL_SET_ERR_MSG_MOD(extack, "Failed to read NVM contents"); - ice_release_nvm(hw); vfree(nvm_data); return -EIO; } - ice_release_nvm(hw); tmp += read_sz; left -= read_sz; @@ -1943,6 +1934,7 @@ static int ice_devlink_nvm_read(struct devlink *devlink, struct netlink_ext_ack *extack, u64 offset, u32 size, u8 *data) { + enum libie_aq_err read_aq_err = LIBIE_AQ_RC_OK; struct ice_pf *pf = devlink_priv(devlink); struct device *dev = ice_pf_to_dev(pf); struct ice_hw *hw = &pf->hw; @@ -1966,24 +1958,14 @@ static int ice_devlink_nvm_read(struct devlink *devlink, return -ERANGE; } - status = ice_acquire_nvm(hw, ICE_RES_READ); - if (status) { - dev_dbg(dev, "ice_acquire_nvm failed, err %d aq_err %d\n", - status, hw->adminq.sq_last_status); - NL_SET_ERR_MSG_MOD(extack, "Failed to acquire NVM semaphore"); - return -EIO; - } - status = ice_read_flat_nvm(hw, (u32)offset, &size, data, - read_shadow_ram); + read_shadow_ram, &read_aq_err); if (status) { dev_dbg(dev, "ice_read_flat_nvm failed after reading %u bytes, err %d aq_err %d\n", - size, status, hw->adminq.sq_last_status); + size, status, read_aq_err); NL_SET_ERR_MSG_MOD(extack, "Failed to read NVM contents"); - ice_release_nvm(hw); return -EIO; } - ice_release_nvm(hw); return 0; } diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index 7eb380be7ed27a..bf9a821c543b7f 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -853,6 +853,7 @@ static int ice_get_eeprom(struct net_device *netdev, struct ethtool_eeprom *eeprom, u8 *bytes) { + enum libie_aq_err read_aq_err = LIBIE_AQ_RC_OK; struct ice_pf *pf = ice_netdev_to_pf(netdev); struct ice_hw *hw = &pf->hw; struct device *dev; @@ -869,24 +870,15 @@ ice_get_eeprom(struct net_device *netdev, struct ethtool_eeprom *eeprom, if (!buf) return -ENOMEM; - ret = ice_acquire_nvm(hw, ICE_RES_READ); - if (ret) { - dev_err(dev, "ice_acquire_nvm failed, err %d aq_err %s\n", - ret, libie_aq_str(hw->adminq.sq_last_status)); - goto out; - } - ret = ice_read_flat_nvm(hw, eeprom->offset, &eeprom->len, buf, - false); + false, &read_aq_err); if (ret) { dev_err(dev, "ice_read_flat_nvm failed, err %d aq_err %s\n", - ret, libie_aq_str(hw->adminq.sq_last_status)); - goto release; + ret, libie_aq_str(read_aq_err)); + goto out; } memcpy(bytes, buf, eeprom->len); -release: - ice_release_nvm(hw); out: kfree(buf); return ret; diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index 8cdc4fda89e978..9e08db376d3d0c 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -2871,6 +2871,9 @@ int ice_vsi_release(struct ice_vsi *vsi) return -ENODEV; pf = vsi->back; + if (ice_is_vsi_dflt_vsi(vsi)) + ice_clear_dflt_vsi(vsi); + if (test_bit(ICE_FLAG_RSS_ENA, pf->flags)) ice_rss_clean(vsi); diff --git a/drivers/net/ethernet/intel/ice/ice_nvm.c b/drivers/net/ethernet/intel/ice/ice_nvm.c index 7e187a804dfa16..21f3b615dbbf19 100644 --- a/drivers/net/ethernet/intel/ice/ice_nvm.c +++ b/drivers/net/ethernet/intel/ice/ice_nvm.c @@ -53,17 +53,27 @@ int ice_aq_read_nvm(struct ice_hw *hw, u16 module_typeid, u32 offset, * @length: (in) number of bytes to read; (out) number of bytes actually read * @data: buffer to return data in (sized to fit the specified length) * @read_shadow_ram: if true, read from shadow RAM instead of NVM + * @read_aq_err: if non-NULL, receives the AQ error status of the failing read * * Reads a portion of the NVM, as a flat memory space. This function correctly * breaks read requests across Shadow RAM sectors and ensures that no single * read request exceeds the maximum 4KB read for a single AdminQ command. * + * FW caps the read lock at a maximum of 3000ms, so a read spanning multiple + * 4KB sectors cannot be done under a single lock without FW reclaiming it + * mid-read. The NVM lock is therefore acquired and released around each AQ + * read, so this function must be called without the lock held. + * + * Since ice_release_nvm() issues an AQ command that overwrites + * hw->adminq.sq_last_status, callers that need the failing read's AQ error + * must use @read_aq_err rather than inspecting sq_last_status afterwards. + * * Returns a status code on failure. Note that the data pointer may be * partially updated if some reads succeed before a failure. */ int ice_read_flat_nvm(struct ice_hw *hw, u32 offset, u32 *length, u8 *data, - bool read_shadow_ram) + bool read_shadow_ram, enum libie_aq_err *read_aq_err) { u32 inlen = *length; u32 bytes_read = 0; @@ -92,12 +102,30 @@ ice_read_flat_nvm(struct ice_hw *hw, u32 offset, u32 *length, u8 *data, last_cmd = !(bytes_read + read_size < inlen); + status = ice_acquire_nvm(hw, ICE_RES_READ); + if (status) { + ice_debug(hw, ICE_DBG_NVM, "Failed to acquire NVM lock, err %d aq_err %s\n", + status, libie_aq_str(hw->adminq.sq_last_status)); + break; + } + status = ice_aq_read_nvm(hw, ICE_AQC_NVM_START_POINT, offset, read_size, data + bytes_read, last_cmd, read_shadow_ram, NULL); - if (status) + if (status) { + /* Capture the read's AQ error before ice_release_nvm() + * issues its own AQ command and overwrites + * sq_last_status. + */ + if (read_aq_err) + *read_aq_err = hw->adminq.sq_last_status; + + ice_release_nvm(hw); break; + } + + ice_release_nvm(hw); bytes_read += read_size; offset += read_size; @@ -177,14 +205,19 @@ int ice_aq_erase_nvm(struct ice_hw *hw, u16 module_typeid, struct ice_sq_cd *cd) } /** - * ice_read_sr_word_aq - Reads Shadow RAM via AQ + * ice_read_sr_word - Reads Shadow RAM word * @hw: pointer to the HW structure * @offset: offset of the Shadow RAM word to read (0x000000 - 0x001FFF) * @data: word read from the Shadow RAM * * Reads one 16 bit word from the Shadow RAM using ice_read_flat_nvm. + * + * The NVM lock is acquired and released internally by ice_read_flat_nvm() + * around the FW read, so this function must be called without the lock held. + * + * Return: zero on success, or a negative error code on failure. */ -static int ice_read_sr_word_aq(struct ice_hw *hw, u16 offset, u16 *data) +int ice_read_sr_word(struct ice_hw *hw, u16 offset, u16 *data) { u32 bytes = sizeof(u16); __le16 data_local; @@ -194,7 +227,7 @@ static int ice_read_sr_word_aq(struct ice_hw *hw, u16 offset, u16 *data) * Shadow RAM sector restrictions necessary when reading from the NVM. */ status = ice_read_flat_nvm(hw, offset * sizeof(u16), &bytes, - (__force u8 *)&data_local, true); + (__force u8 *)&data_local, true, NULL); if (status) return status; @@ -330,13 +363,8 @@ ice_read_flash_module(struct ice_hw *hw, enum ice_bank_select bank, u16 module, return -EINVAL; } - status = ice_acquire_nvm(hw, ICE_RES_READ); - if (status) - return status; - - status = ice_read_flat_nvm(hw, start + offset, &length, data, false); - - ice_release_nvm(hw); + status = ice_read_flat_nvm(hw, start + offset, &length, data, false, + NULL); return status; } @@ -418,27 +446,6 @@ ice_read_netlist_module(struct ice_hw *hw, enum ice_bank_select bank, u32 offset return status; } -/** - * ice_read_sr_word - Reads Shadow RAM word and acquire NVM if necessary - * @hw: pointer to the HW structure - * @offset: offset of the Shadow RAM word to read (0x000000 - 0x001FFF) - * @data: word read from the Shadow RAM - * - * Reads one 16 bit word from the Shadow RAM using the ice_read_sr_word_aq. - */ -int ice_read_sr_word(struct ice_hw *hw, u16 offset, u16 *data) -{ - int status; - - status = ice_acquire_nvm(hw, ICE_RES_READ); - if (!status) { - status = ice_read_sr_word_aq(hw, offset, data); - ice_release_nvm(hw); - } - - return status; -} - /** * ice_get_pfa_module_tlv - Reads sub module TLV from NVM PFA * @hw: pointer to hardware structure @@ -856,20 +863,18 @@ int ice_get_inactive_netlist_ver(struct ice_hw *hw, struct ice_netlist_info *net static int ice_discover_flash_size(struct ice_hw *hw) { u32 min_size = 0, max_size = ICE_AQC_NVM_MAX_OFFSET + 1; - int status; - - status = ice_acquire_nvm(hw, ICE_RES_READ); - if (status) - return status; + int status = 0; while ((max_size - min_size) > 1) { + enum libie_aq_err read_aq_err = LIBIE_AQ_RC_OK; u32 offset = (max_size + min_size) / 2; u32 len = 1; u8 data; - status = ice_read_flat_nvm(hw, offset, &len, &data, false); + status = ice_read_flat_nvm(hw, offset, &len, &data, false, + &read_aq_err); if (status == -EIO && - hw->adminq.sq_last_status == LIBIE_AQ_RC_EINVAL) { + read_aq_err == LIBIE_AQ_RC_EINVAL) { ice_debug(hw, ICE_DBG_NVM, "%s: New upper bound of %u bytes\n", __func__, offset); status = 0; @@ -880,7 +885,7 @@ static int ice_discover_flash_size(struct ice_hw *hw) min_size = offset; } else { /* an unexpected error occurred */ - goto err_read_flat_nvm; + return status; } } @@ -888,9 +893,6 @@ static int ice_discover_flash_size(struct ice_hw *hw) hw->flash.flash_size = max_size; -err_read_flat_nvm: - ice_release_nvm(hw); - return status; } diff --git a/drivers/net/ethernet/intel/ice/ice_nvm.h b/drivers/net/ethernet/intel/ice/ice_nvm.h index 63cdc6bdac5899..e1d1a11f5ca419 100644 --- a/drivers/net/ethernet/intel/ice/ice_nvm.h +++ b/drivers/net/ethernet/intel/ice/ice_nvm.h @@ -19,7 +19,7 @@ int ice_aq_read_nvm(struct ice_hw *hw, u16 module_typeid, u32 offset, bool read_shadow_ram, struct ice_sq_cd *cd); int ice_read_flat_nvm(struct ice_hw *hw, u32 offset, u32 *length, u8 *data, - bool read_shadow_ram); + bool read_shadow_ram, enum libie_aq_err *read_aq_err); int ice_get_pfa_module_tlv(struct ice_hw *hw, u16 *module_tlv, u16 *module_tlv_len, u16 module_type); diff --git a/drivers/net/ethernet/intel/ice/ice_ptp_hw.c b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c index 8e5f97835954c3..3a41c711e751b5 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp_hw.c +++ b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c @@ -4808,15 +4808,12 @@ static int ice_ptp_prep_phy_adj_ll_e810(struct ice_hw *hw, s32 adj) !FIELD_GET(REG_LL_PROXY_H_EXEC, val), 10, REG_LL_PROXY_H_TIMEOUT_US, false, hw, REG_LL_PROXY_H); - if (err) { - ice_debug(hw, ICE_DBG_PTP, "Failed to prepare PHY timer adjustment using low latency interface\n"); - spin_unlock_irq(¶ms->atqbal_wq.lock); - return err; - } - spin_unlock_irq(¶ms->atqbal_wq.lock); - return 0; + if (err) + ice_debug(hw, ICE_DBG_PTP, "Failed to prepare PHY timer adjustment using low latency interface\n"); + + return err; } /** @@ -4837,8 +4834,12 @@ static int ice_ptp_prep_phy_adj_e810(struct ice_hw *hw, s32 adj) u8 tmr_idx; int err; - if (hw->dev_caps.ts_dev_info.ll_phy_tmr_update) - return ice_ptp_prep_phy_adj_ll_e810(hw, adj); + if (hw->dev_caps.ts_dev_info.ll_phy_tmr_update) { + err = ice_ptp_prep_phy_adj_ll_e810(hw, adj); + if (err != -ETIMEDOUT) + return err; + ice_debug(hw, ICE_DBG_PTP, "LL adj timed out, falling back to SBQ\n"); + } tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; @@ -4901,15 +4902,12 @@ static int ice_ptp_prep_phy_incval_ll_e810(struct ice_hw *hw, u64 incval) !FIELD_GET(REG_LL_PROXY_H_EXEC, val), 10, REG_LL_PROXY_H_TIMEOUT_US, false, hw, REG_LL_PROXY_H); - if (err) { - ice_debug(hw, ICE_DBG_PTP, "Failed to prepare PHY timer increment using low latency interface\n"); - spin_unlock_irq(¶ms->atqbal_wq.lock); - return err; - } - spin_unlock_irq(¶ms->atqbal_wq.lock); - return 0; + if (err) + ice_debug(hw, ICE_DBG_PTP, "Failed to prepare PHY timer increment using low latency interface\n"); + + return err; } /** @@ -4927,8 +4925,12 @@ static int ice_ptp_prep_phy_incval_e810(struct ice_hw *hw, u64 incval) u8 tmr_idx; int err; - if (hw->dev_caps.ts_dev_info.ll_phy_tmr_update) - return ice_ptp_prep_phy_incval_ll_e810(hw, incval); + if (hw->dev_caps.ts_dev_info.ll_phy_tmr_update) { + err = ice_ptp_prep_phy_incval_ll_e810(hw, incval); + if (err != -ETIMEDOUT) + return err; + ice_debug(hw, ICE_DBG_PTP, "LL incval timed out, falling back to SBQ\n"); + } tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; low = lower_32_bits(incval); diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index c724d429a7aa98..91ca75e454630f 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -2408,7 +2408,7 @@ void idpf_tx_splitq_build_flow_desc(union idpf_tx_flex_desc *desc, struct idpf_tx_splitq_params *params, u16 td_cmd, u16 size) { - *(u32 *)&desc->flow.qw1.cmd_dtype = (u8)(params->dtype | td_cmd); + *(__le32 *)&desc->flow.qw1.cmd_dtype = cpu_to_le32((u8)(params->dtype | td_cmd)); desc->flow.qw1.rxr_bufsize = cpu_to_le16((u16)size); desc->flow.qw1.compl_tag = cpu_to_le16(params->compl_tag); } diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 39043d8ca1bf48..59067939a586dc 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -3074,7 +3074,8 @@ static void igc_xdp_xmit_zc(struct igc_ring *ring) olinfo_status = xdp_desc.len << IGC_ADVTXD_PAYLEN_SHIFT; dma = xsk_buff_raw_get_dma(pool, xdp_desc.addr); - meta = xsk_buff_get_metadata(pool, xdp_desc.addr); + meta = xsk_buff_get_metadata(pool, xdp_desc.addr, + xdp_desc.options); xsk_buff_raw_dma_sync_for_device(pool, dma, xdp_desc.len); bi = &ring->tx_buffer_info[ntu]; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/api.h b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/api.h index 4285b5d6a6a29a..f36a1d5f236f61 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/api.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/api.h @@ -21,6 +21,7 @@ int cn20k_rvu_mbox_init(struct rvu *rvu, int type, int num); int cn20k_rvu_get_mbox_regions(struct rvu *rvu, void **mbox_addr, int num, int type, unsigned long *pf_bmap); void cn20k_free_mbox_memory(struct rvu *rvu); +void cn20k_free_mbox_memory_type(struct rvu *rvu, int type); int cn20k_register_afpf_mbox_intr(struct rvu *rvu); int cn20k_register_afvf_mbox_intr(struct rvu *rvu, int pf_vec_start); void cn20k_rvu_enable_mbox_intr(struct rvu *rvu); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c index 71401dec0d776b..01f32adac59922 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c @@ -335,13 +335,30 @@ int cn20k_rvu_mbox_init(struct rvu *rvu, int type, int ndevs) return rvu_alloc_mbox_memory(rvu, type, ndevs, MBOX_SIZE); } +void cn20k_free_mbox_memory_type(struct rvu *rvu, int type) +{ + if (!is_cn20k(rvu->pdev) || !rvu->ng_rvu) + return; + + switch (type) { + case TYPE_AFPF: + qmem_free(rvu->dev, rvu->ng_rvu->pf_mbox_addr); + rvu->ng_rvu->pf_mbox_addr = NULL; + break; + case TYPE_AFVF: + qmem_free(rvu->dev, rvu->ng_rvu->vf_mbox_addr); + rvu->ng_rvu->vf_mbox_addr = NULL; + break; + } +} + void cn20k_free_mbox_memory(struct rvu *rvu) { if (!is_cn20k(rvu->pdev)) return; - qmem_free(rvu->dev, rvu->ng_rvu->pf_mbox_addr); - qmem_free(rvu->dev, rvu->ng_rvu->vf_mbox_addr); + cn20k_free_mbox_memory_type(rvu, TYPE_AFPF); + cn20k_free_mbox_memory_type(rvu, TYPE_AFVF); } void cn20k_rvu_disable_afvf_intr(struct rvu *rvu, int vfs) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c index d98b49f47970ba..fce22e314cac09 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c @@ -856,7 +856,7 @@ int rvu_mbox_handler_mcs_ctrl_pkt_rule_write(struct rvu *rvu, static void rvu_mcs_set_lmac_bmap(struct rvu *rvu) { struct mcs *mcs = mcs_get_pdata(0); - unsigned long lmac_bmap; + unsigned long lmac_bmap = 0; int cgx, lmac, port; for (port = 0; port < mcs->hw->lmac_cnt; port++) { diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c index ffba56ee8a60cd..74c041ab5280a3 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c @@ -2585,12 +2585,6 @@ static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw, if (!pf_bmap) return -ENOMEM; - ng_rvu_mbox = kzalloc_obj(*ng_rvu_mbox); - if (!ng_rvu_mbox) { - err = -ENOMEM; - goto free_bitmap; - } - /* RVU VFs */ if (type == TYPE_AFVF) bitmap_set(pf_bmap, 0, num); @@ -2604,15 +2598,22 @@ static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw, } } - rvu->ng_rvu = ng_rvu_mbox; + if (!rvu->ng_rvu) { + ng_rvu_mbox = devm_kzalloc(rvu->dev, sizeof(*ng_rvu_mbox), GFP_KERNEL); + if (!ng_rvu_mbox) { + err = -ENOMEM; + goto free_bitmap; + } + + rvu->ng_rvu = ng_rvu_mbox; - rvu->ng_rvu->rvu_mbox_ops = &rvu_mbox_ops; + rvu->ng_rvu->rvu_mbox_ops = &rvu_mbox_ops; + mutex_init(&rvu->mbox_lock); + } err = cn20k_rvu_mbox_init(rvu, type, num); if (err) - goto free_mem; - - mutex_init(&rvu->mbox_lock); + goto free_bitmap; mbox_regions = kcalloc(num, sizeof(void __iomem *), GFP_KERNEL); if (!mbox_regions) { @@ -2702,14 +2703,18 @@ unmap_regions: free_regions: kfree(mbox_regions); free_qmem: - cn20k_free_mbox_memory(rvu); -free_mem: - kfree(rvu->ng_rvu); + cn20k_free_mbox_memory_type(rvu, type); free_bitmap: bitmap_free(pf_bmap); return err; } +static void rvu_free_cn20k_mbox_memory(struct rvu *rvu) +{ + if (is_cn20k(rvu->pdev)) + cn20k_free_mbox_memory(rvu); +} + static void rvu_mbox_destroy(struct mbox_wq_info *mw) { struct otx2_mbox *mbox = &mw->mbox; @@ -3331,8 +3336,8 @@ static int rvu_register_interrupts(struct rvu *rvu) goto fail; for (i = 0; i < rvu->num_vec; i++) { - if (strstr(&rvu->irq_name[i * NAME_SIZE], "Mbox") || - strstr(&rvu->irq_name[i * NAME_SIZE], "FLR")) + if (strnstr(&rvu->irq_name[i * NAME_SIZE], "Mbox", NAME_SIZE) || + strnstr(&rvu->irq_name[i * NAME_SIZE], "FLR", NAME_SIZE)) irq_set_affinity(pci_irq_vector(rvu->pdev, i), cpumask_of(0)); } @@ -3519,6 +3524,7 @@ static int rvu_enable_sriov(struct rvu *rvu) if (err) { rvu_disable_afvf_intr(rvu); rvu_mbox_destroy(&rvu->afvf_wq_info); + cn20k_free_mbox_memory_type(rvu, TYPE_AFVF); return err; } @@ -3681,6 +3687,7 @@ err_flr: err_mbox: rvu_mbox_destroy(&rvu->afpf_wq_info); err_hwsetup: + rvu_free_cn20k_mbox_memory(rvu); rvu_cgx_exit(rvu); rvu_fwdata_exit(rvu); rvu_mcs_exit(rvu); @@ -3723,9 +3730,7 @@ static void rvu_remove(struct pci_dev *pdev) pci_set_drvdata(pdev, NULL); devm_kfree(&pdev->dev, rvu->hw); - if (is_cn20k(rvu->pdev)) - cn20k_free_mbox_memory(rvu); - kfree(rvu->ng_rvu); + rvu_free_cn20k_mbox_memory(rvu); devm_kfree(&pdev->dev, rvu); atomic_set(&device_bound, 0); } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c index 3456313d3b3c06..22ee9967687963 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c @@ -1697,6 +1697,12 @@ static int rvu_dbg_nix_tm_tree_display(struct seq_file *m, void *unused) return -EINVAL; pfvf = rvu_get_pfvf(rvu, pcifunc); + + if (!pfvf->sq_ctx) { + seq_printf(m, "SQ context is not initialized for pcifunc 0x%x\n", pcifunc); + return -EINVAL; + } + max_id = pfvf->sq_ctx->qsize; memset(&aq_req, 0, sizeof(struct nix_aq_enq_req)); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c index 964bcaae098e25..cee30d5f1e1e5d 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c @@ -2483,8 +2483,8 @@ static int nix_smq_flush(struct rvu *rvu, int blkaddr, int pf = rvu_get_pf(rvu->pdev, pcifunc); u8 cgx_id = 0, lmac_id = 0; u16 tl2_tl3_link_schq; - u8 link, link_level; u64 cfg, bmap = 0; + u8 link_level; if (!is_rvu_otx2(rvu)) { /* Skip SMQ flush if pkt count is zero */ @@ -2516,7 +2516,6 @@ static int nix_smq_flush(struct rvu *rvu, int blkaddr, link_level = rvu_read64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL) & 0x01 ? NIX_TXSCH_LVL_TL3 : NIX_TXSCH_LVL_TL2; tl2_tl3_link_schq = smq_flush_ctx->smq_tree_ctx[link_level].schq; - link = smq_flush_ctx->smq_tree_ctx[NIX_TXSCH_LVL_TL1].schq; /* SMQ set enqueue xoff */ cfg = rvu_read64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq)); @@ -2526,13 +2525,13 @@ static int nix_smq_flush(struct rvu *rvu, int blkaddr, /* Clear all NIX_AF_TL3_TL2_LINK_CFG[ENA] for the TL3/TL2 queue */ for (i = 0; i < (rvu->hw->cgx_links + rvu->hw->lbk_links); i++) { cfg = rvu_read64(rvu, blkaddr, - NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link)); + NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, i)); if (!(cfg & BIT_ULL(12))) continue; bmap |= BIT_ULL(i); cfg &= ~BIT_ULL(12); rvu_write64(rvu, blkaddr, - NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link), cfg); + NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, i), cfg); } /* Do SMQ flush and set enqueue xoff */ @@ -2553,10 +2552,10 @@ static int nix_smq_flush(struct rvu *rvu, int blkaddr, if (!(bmap & BIT_ULL(i))) continue; cfg = rvu_read64(rvu, blkaddr, - NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link)); + NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, i)); cfg |= BIT_ULL(12); rvu_write64(rvu, blkaddr, - NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link), cfg); + NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, i), cfg); } /* clear XOFF on TL2s */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c index 3d253132a17f0e..52980c4afb54b1 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c @@ -333,7 +333,8 @@ int otx2_set_rss_table(struct otx2_nic *pfvf, int ctx_id, const u32 *ind_tbl) /* Get memory to put this msg */ for (idx = 0; idx < rss->rss_size; idx++) { /* Ignore the queue if AF_XDP zero copy is enabled */ - if (test_bit(ind_tbl[idx], pfvf->af_xdp_zc_qidx)) + if (pfvf->af_xdp_zc_qidx && + test_bit(ind_tbl[idx], pfvf->af_xdp_zc_qidx)) continue; aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox); @@ -1510,7 +1511,8 @@ int otx2_pool_aq_init(struct otx2_nic *pfvf, u16 pool_id, if (type != AURA_NIX_RQ) return 0; - if (!test_bit(pool_id, pfvf->af_xdp_zc_qidx)) { + if (!pfvf->af_xdp_zc_qidx || + !test_bit(pool_id, pfvf->af_xdp_zc_qidx)) { pp_params.order = get_order(buf_size); pp_params.flags = PP_FLAG_DMA_MAP; pp_params.pool_size = min(OTX2_PAGE_POOL_SZ, numptrs); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c index a0340f3422bf90..9bee1b91eeaa67 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c @@ -939,7 +939,8 @@ static int otx2_get_rxfh(struct net_device *dev, for (idx = 0; idx < rss->rss_size; idx++) { /* Ignore if the rx queue is AF_XDP zero copy enabled */ - if (test_bit(rss->ind_tbl[idx], pfvf->af_xdp_zc_qidx)) + if (pfvf->af_xdp_zc_qidx && + test_bit(rss->ind_tbl[idx], pfvf->af_xdp_zc_qidx)) continue; indir[idx] = rss->ind_tbl[idx]; } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c index b022f52c684508..fcdf891f90b580 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c @@ -737,16 +737,16 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (err) goto err_ptp_destroy; + err = otx2_vf_wq_init(vf); + if (err) + goto err_ipsec_clean; + err = register_netdev(netdev); if (err) { dev_err(dev, "Failed to register netdevice\n"); - goto err_ipsec_clean; + goto err_wq_destroy; } - err = otx2_vf_wq_init(vf); - if (err) - goto err_unreg_netdev; - otx2vf_set_ethtool_ops(netdev); err = otx2vf_mcam_flow_init(vf); @@ -789,6 +789,10 @@ err_shutdown_tc: otx2_shutdown_tc(vf); err_unreg_netdev: unregister_netdev(netdev); +err_wq_destroy: + cancel_work_sync(&vf->reset_task); + cancel_work_sync(&vf->rx_mode_work); + destroy_workqueue(vf->otx2_wq); err_ipsec_clean: cn10k_ipsec_clean(vf); err_ptp_destroy: @@ -836,11 +840,13 @@ static void otx2vf_remove(struct pci_dev *pdev) } #endif - cancel_work_sync(&vf->reset_task); otx2_unregister_dl(vf); unregister_netdev(netdev); - if (vf->otx2_wq) + if (vf->otx2_wq) { + cancel_work_sync(&vf->reset_task); + cancel_work_sync(&vf->rx_mode_work); destroy_workqueue(vf->otx2_wq); + } cn10k_ipsec_clean(vf); otx2_ptp_destroy(vf); otx2_mcam_flow_del(vf); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c index 7d67b4cbaf7161..0e8a6a6486c4ac 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c @@ -193,7 +193,8 @@ int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) void otx2_attach_xsk_buff(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, int qidx) { - if (test_bit(qidx, pfvf->af_xdp_zc_qidx)) + if (pfvf->af_xdp_zc_qidx && + test_bit(qidx, pfvf->af_xdp_zc_qidx)) sq->xsk_pool = xsk_get_pool_from_qid(pfvf->netdev, qidx); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c index 3d19dad8f86889..d194eeddd7b55a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c @@ -99,7 +99,7 @@ bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget) xdptxd.dma_addr = xsk_buff_raw_get_dma(pool, desc.addr); xdptxd.data = xsk_buff_raw_get_data(pool, desc.addr); xdptxd.len = desc.len; - meta = xsk_buff_get_metadata(pool, desc.addr); + meta = xsk_buff_get_metadata(pool, desc.addr, desc.options); xsk_buff_raw_dma_sync_for_device(pool, xdptxd.dma_addr, xdptxd.len); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index c8b76d301c92cf..3d544fe4e6f73b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -689,11 +689,13 @@ static int mlx5e_rep_open(struct net_device *dev) if (err) goto unlock; + mutex_lock(&rep->esw->state_lock); if (!mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, rep->vport, 1, MLX5_VPORT_ADMIN_STATE_UP)) netif_carrier_on(dev); + mutex_unlock(&rep->esw->state_lock); unlock: mutex_unlock(&priv->state_lock); @@ -708,10 +710,12 @@ static int mlx5e_rep_close(struct net_device *dev) int ret; mutex_lock(&priv->state_lock); + mutex_lock(&rep->esw->state_lock); mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, rep->vport, 1, MLX5_VPORT_ADMIN_STATE_DOWN); + mutex_unlock(&rep->esw->state_lock); ret = mlx5e_close_locked(dev); mutex_unlock(&priv->state_lock); return ret; @@ -783,22 +787,25 @@ static int mlx5e_rep_change_carrier(struct net_device *dev, bool new_carrier) struct mlx5e_priv *priv = netdev_priv(dev); struct mlx5e_rep_priv *rpriv = priv->ppriv; struct mlx5_eswitch_rep *rep = rpriv->rep; - int err; + int err = 0; + mutex_lock(&rep->esw->state_lock); if (new_carrier) { err = mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, rep->vport, 1, MLX5_VPORT_ADMIN_STATE_UP); if (err) - return err; + goto unlock; netif_carrier_on(dev); } else { err = mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, rep->vport, 1, MLX5_VPORT_ADMIN_STATE_DOWN); if (err) - return err; + goto unlock; netif_carrier_off(dev); } - return 0; +unlock: + mutex_unlock(&rep->esw->state_lock); + return err; } static const struct net_device_ops mlx5e_netdev_ops_rep = { @@ -1337,9 +1344,12 @@ static void mlx5e_uplink_rep_enable(struct mlx5e_priv *priv) mlx5e_rep_tc_enable(priv); - if (MLX5_CAP_GEN(mdev, uplink_follow)) + if (MLX5_CAP_GEN(mdev, uplink_follow)) { + mutex_lock(&mdev->priv.eswitch->state_lock); mlx5_modify_vport_admin_state(mdev, MLX5_VPORT_STATE_OP_MOD_UPLINK, 0, 0, MLX5_VPORT_ADMIN_STATE_AUTO); + mutex_unlock(&mdev->priv.eswitch->state_lock); + } mlx5_lag_add_netdev(mdev, netdev); priv->events_nb.notifier_call = uplink_rep_async_event; mlx5_notifier_register(mdev, &priv->events_nb); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/adj_vport.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/adj_vport.c index ca249b50f830a5..7950b82d8b8aec 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/adj_vport.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/adj_vport.c @@ -9,6 +9,28 @@ int mlx5_esw_adj_vport_modify(struct mlx5_core_dev *dev, u16 vport, { u32 in[MLX5_ST_SZ_DW(modify_vport_state_in)] = {}; + lockdep_assert_held(&dev->priv.eswitch->state_lock); + + if (MLX5_CAP_ESW(dev, esw_vport_state_max_tx_speed)) { + u8 op_mod = MLX5_VPORT_STATE_OP_MOD_ESW_VPORT; + struct mlx5_vport *esw_vport; + u32 speed = 0; + int err; + + err = mlx5_query_vport_max_tx_speed(dev, op_mod, vport, + true, &speed, NULL); + if (err) { + esw_vport = mlx5_eswitch_get_vport(dev->priv.eswitch, + vport); + speed = IS_ERR(esw_vport) ? 0 : + esw_vport->agg_max_tx_speed; + mlx5_core_dbg(dev, + "Failed to query vport %d max tx speed, err=%d, using cached %u\n", + vport, err, speed); + } + MLX5_SET(modify_vport_state_in, in, max_tx_speed, speed); + } + MLX5_SET(modify_vport_state_in, in, opcode, MLX5_CMD_OP_MODIFY_VPORT_STATE); MLX5_SET(modify_vport_state_in, in, op_mod, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 247e5d85ec3671..d603e294ee0e88 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -2567,6 +2567,7 @@ static void mlx5_esw_fdb_active(struct mlx5_eswitch *esw) mlx5_esw_fdb_drop_destroy(esw); mlx5_mpfs_enable(esw->dev); + mutex_lock(&esw->state_lock); mlx5_esw_for_each_vf_vport(esw, i, vport, U16_MAX) { if (!vport->adjacent) continue; @@ -2574,6 +2575,7 @@ static void mlx5_esw_fdb_active(struct mlx5_eswitch *esw) vport->vport); mlx5_esw_adj_vport_modify(esw->dev, vport->vport, true); } + mutex_unlock(&esw->state_lock); esw->offloads_inactive = false; esw_warn(esw->dev, "MPFS/FDB active\n"); @@ -2587,6 +2589,7 @@ static void mlx5_esw_fdb_inactive(struct mlx5_eswitch *esw) mlx5_mpfs_disable(esw->dev); mlx5_esw_fdb_drop_create(esw); + mutex_lock(&esw->state_lock); mlx5_esw_for_each_vf_vport(esw, i, vport, U16_MAX) { if (!vport->adjacent) continue; @@ -2595,6 +2598,7 @@ static void mlx5_esw_fdb_inactive(struct mlx5_eswitch *esw) mlx5_esw_adj_vport_modify(esw->dev, vport->vport, false); } + mutex_unlock(&esw->state_lock); esw->offloads_inactive = true; esw_warn(esw->dev, "MPFS/FDB inactive\n"); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index 28d16fdc3f0615..2285c889c215e4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -1471,6 +1471,7 @@ static void mlx5_lag_modify_device_vports_speed(struct mlx5_core_dev *mdev, if (!MLX5_CAP_ESW(mdev, esw_vport_state_max_tx_speed)) return; + mutex_lock(&esw->state_lock); mlx5_esw_for_each_vport(esw, i, vport) { if (!vport) continue; @@ -1490,6 +1491,7 @@ static void mlx5_lag_modify_device_vports_speed(struct mlx5_core_dev *mdev, "Failed to set vport %d speed %d, err=%d\n", vport->vport, speed, ret); } + mutex_unlock(&esw->state_lock); } void mlx5_lag_set_vports_agg_speed(struct mlx5_lag *ldev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c index 3676e26ac6b09b..8aa94ec87a0e3c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c @@ -89,6 +89,34 @@ int mlx5_modify_vport_admin_state(struct mlx5_core_dev *mdev, u8 opmod, { u32 in[MLX5_ST_SZ_DW(modify_vport_state_in)] = {}; +#ifdef CONFIG_MLX5_ESWITCH + lockdep_assert_held(&mdev->priv.eswitch->state_lock); +#endif + + if (MLX5_CAP_ESW(mdev, esw_vport_state_max_tx_speed) && + opmod == MLX5_VPORT_STATE_OP_MOD_ESW_VPORT && + vport != MLX5_VPORT_UPLINK) { + u32 speed = 0; + int err; + + err = mlx5_query_vport_max_tx_speed(mdev, opmod, vport, + other_vport, &speed, NULL); + if (err) { +#ifdef CONFIG_MLX5_ESWITCH + struct mlx5_vport *esw_vport; + + esw_vport = mlx5_eswitch_get_vport(mdev->priv.eswitch, + vport); + speed = IS_ERR(esw_vport) ? 0 : + esw_vport->agg_max_tx_speed; +#endif + mlx5_core_dbg(mdev, + "Failed to query vport %d max tx speed, err=%d, using cached %u\n", + vport, err, speed); + } + MLX5_SET(modify_vport_state_in, in, max_tx_speed, speed); + } + MLX5_SET(modify_vport_state_in, in, opcode, MLX5_CMD_OP_MODIFY_VPORT_STATE); MLX5_SET(modify_vport_state_in, in, op_mod, opmod); @@ -106,6 +134,10 @@ int mlx5_modify_vport_max_tx_speed(struct mlx5_core_dev *mdev, u8 opmod, u8 admin_state; int err; +#ifdef CONFIG_MLX5_ESWITCH + lockdep_assert_held(&mdev->priv.eswitch->state_lock); +#endif + err = mlx5_query_vport_admin_state(mdev, opmod, vport, other_vport, &admin_state); if (err) @@ -123,11 +155,11 @@ int mlx5_modify_vport_max_tx_speed(struct mlx5_core_dev *mdev, u8 opmod, } int mlx5_query_vport_max_tx_speed(struct mlx5_core_dev *mdev, u8 op_mod, - u16 vport, u8 other_vport, u32 *max_tx_speed) + u16 vport, u8 other_vport, + u32 *max_tx_speed, u8 *state) { u32 out[MLX5_ST_SZ_DW(query_vport_state_out)] = {}; u32 in[MLX5_ST_SZ_DW(query_vport_state_in)] = {}; - u32 state; int err; MLX5_SET(query_vport_state_in, in, opcode, @@ -140,13 +172,9 @@ int mlx5_query_vport_max_tx_speed(struct mlx5_core_dev *mdev, u8 op_mod, if (err) return err; - state = MLX5_GET(query_vport_state_out, out, state); - if (state == VPORT_STATE_DOWN) { - *max_tx_speed = 0; - return 0; - } - *max_tx_speed = MLX5_GET(query_vport_state_out, out, max_tx_speed); + if (state) + *state = MLX5_GET(query_vport_state_out, out, state); return 0; } EXPORT_SYMBOL_GPL(mlx5_query_vport_max_tx_speed); diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c b/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c index 1d34af78166a3e..4dedf5d9264167 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c @@ -162,13 +162,18 @@ static int sparx5_port_stop(struct net_device *ndev) return 0; } -static void sparx5_set_rx_mode(struct net_device *dev) +static int sparx5_set_rx_mode(struct net_device *dev, + struct netdev_hw_addr_list *uc, + struct netdev_hw_addr_list *mc) { struct sparx5_port *port = netdev_priv(dev); struct sparx5 *sparx5 = port->sparx5; if (!test_bit(port->portno, sparx5->bridge_mask)) - __dev_mc_sync(dev, sparx5_mc_sync, sparx5_mc_unsync); + return __hw_addr_sync_dev(mc, dev, sparx5_mc_sync, + sparx5_mc_unsync); + + return 0; } static int sparx5_port_get_phys_port_name(struct net_device *dev, @@ -249,7 +254,7 @@ static const struct net_device_ops sparx5_port_netdev_ops = { .ndo_open = sparx5_port_open, .ndo_stop = sparx5_port_stop, .ndo_start_xmit = sparx5_port_xmit_impl, - .ndo_set_rx_mode = sparx5_set_rx_mode, + .ndo_set_rx_mode_async = sparx5_set_rx_mode, .ndo_get_phys_port_name = sparx5_port_get_phys_port_name, .ndo_set_mac_address = sparx5_set_mac_address, .ndo_validate_addr = eth_validate_addr, diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c b/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c index cf332de6bf736c..2dee2ce19fce74 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c @@ -2077,8 +2077,9 @@ int sparx5_vcap_init(struct sparx5 *sparx5) dir = vcap_debugfs(sparx5->dev, sparx5->debugfs_root, ctrl); for (idx = 0; idx < consts->n_ports; ++idx) if (sparx5->ports[idx]) - vcap_port_debugfs(sparx5->dev, dir, ctrl, - sparx5->ports[idx]->ndev); + vcap_port_debugfs_portno(sparx5->dev, dir, ctrl, + sparx5->ports[idx]->ndev, + sparx5->ports[idx]->portno); return err; } diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.c b/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.c index e0c65c7ab23e1e..476f7496a9d449 100644 --- a/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.c +++ b/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.c @@ -400,6 +400,27 @@ void vcap_port_debugfs(struct device *dev, struct dentry *parent, } EXPORT_SYMBOL_GPL(vcap_port_debugfs); +void vcap_port_debugfs_portno(struct device *dev, + struct dentry *parent, + struct vcap_control *vctrl, + struct net_device *ndev, + unsigned int portno) +{ + struct vcap_port_debugfs_info *info; + char name[16]; + + info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL); + if (!info) + return; + + info->vctrl = vctrl; + info->ndev = ndev; + + snprintf(name, sizeof(name), "p%u", portno); + debugfs_create_file(name, 0444, parent, info, &vcap_port_debugfs_fops); +} +EXPORT_SYMBOL_GPL(vcap_port_debugfs_portno); + /* Show the full VCAP instance data (rules with all fields) */ static int vcap_debugfs_show(struct seq_file *m, void *unused) { diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.h b/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.h index 9f2c59b5f6f530..7dc6e3411a4d94 100644 --- a/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.h +++ b/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.h @@ -18,6 +18,12 @@ void vcap_port_debugfs(struct device *dev, struct dentry *parent, struct vcap_control *vctrl, struct net_device *ndev); +void vcap_port_debugfs_portno(struct device *dev, + struct dentry *parent, + struct vcap_control *vctrl, + struct net_device *ndev, + unsigned int portno); + /* Create a debugFS entry for a vcap instance */ struct dentry *vcap_debugfs(struct device *dev, struct dentry *parent, struct vcap_control *vctrl); @@ -30,6 +36,14 @@ static inline void vcap_port_debugfs(struct device *dev, struct dentry *parent, { } +static inline void vcap_port_debugfs_portno(struct device *dev, + struct dentry *parent, + struct vcap_control *vctrl, + struct net_device *ndev, + unsigned int portno) +{ +} + static inline struct dentry *vcap_debugfs(struct device *dev, struct dentry *parent, struct vcap_control *vctrl) diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index e8b7ffb47eb982..6a150a89380062 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -180,6 +180,7 @@ static int mana_gd_query_max_resources(struct pci_dev *pdev) struct gdma_query_max_resources_resp resp = {}; struct gdma_general_req req = {}; unsigned int max_num_queues; + unsigned int msix_vec_count; u8 bm_hostmode; u16 num_ports; int err; @@ -216,6 +217,24 @@ static int mana_gd_query_max_resources(struct pci_dev *pdev) gc->num_msix_usable = min(resp.max_msix, num_online_cpus() + 1); } + /* MSI-X vectors are allocated by index into the device MSI-X table, so + * never ask for more than the table holds. It can be smaller than both + * resp.max_msix and the CPU count. + */ + err = pci_msix_vec_count(pdev); + if (err <= 0) { + dev_err(gc->dev, "Failed to query MSI-X table size: %d\n", err); + return err < 0 ? err : -ENOSPC; + } + msix_vec_count = err; + + if (gc->num_msix_usable > msix_vec_count) { + dev_info(gc->dev, + "Limiting MSI-X vectors from %u to table size %u\n", + gc->num_msix_usable, msix_vec_count); + gc->num_msix_usable = msix_vec_count; + } + if (gc->num_msix_usable <= 1) return -ENOSPC; diff --git a/drivers/net/ethernet/nvidia/forcedeth.c b/drivers/net/ethernet/nvidia/forcedeth.c index dc804e11156415..f0218a0eab5ca3 100644 --- a/drivers/net/ethernet/nvidia/forcedeth.c +++ b/drivers/net/ethernet/nvidia/forcedeth.c @@ -2740,7 +2740,7 @@ static void nv_tx_timeout(struct net_device *dev, unsigned int txqueue) netdev_info(dev, "Ring at %lx\n", (unsigned long)np->ring_addr); netdev_info(dev, "Dumping tx registers\n"); - for (i = 0; i <= np->register_size; i += 32) { + for (i = 0; i + 32 <= np->register_size; i += 32) { netdev_info(dev, "%3x: %08x %08x %08x %08x " "%08x %08x %08x %08x\n", diff --git a/drivers/net/ethernet/pensando/ionic/ionic_main.c b/drivers/net/ethernet/pensando/ionic/ionic_main.c index 6e6f3ed07271b3..10501be9ef956b 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_main.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_main.c @@ -269,6 +269,8 @@ bool ionic_notifyq_service(struct ionic_cq *cq) if ((s64)(eid - lif->last_eid) <= 0) return false; + dma_rmb(); + lif->last_eid = eid; dev_dbg(lif->ionic->dev, "notifyq event:\n"); @@ -314,6 +316,8 @@ bool ionic_adminq_service(struct ionic_cq *cq) if (!color_match(comp->color, cq->done_color)) return false; + dma_rmb(); + /* check for empty queue */ if (q->tail_idx == q->head_idx) return false; diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c index 73998d61593a97..e436e3231e864b 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c @@ -706,11 +706,7 @@ static void ionic_rx_clean(struct ionic_queue *q, __le64 *cq_desc_hwstamp; u64 hwstamp; - cq_desc_hwstamp = - (void *)comp + - qcq->cq.desc_size - - sizeof(struct ionic_rxq_comp) - - IONIC_HWSTAMP_CQ_NEGOFFSET; + cq_desc_hwstamp = (void *)comp - IONIC_HWSTAMP_CQ_NEGOFFSET; hwstamp = le64_to_cpu(*cq_desc_hwstamp); @@ -734,11 +730,18 @@ static bool __ionic_rx_service(struct ionic_cq *cq, struct bpf_prog *xdp_prog) struct ionic_queue *q = cq->bound_q; struct ionic_rxq_comp *comp; - comp = &((struct ionic_rxq_comp *)cq->base)[cq->tail_idx]; + if (likely(cq->desc_size == sizeof(*comp))) + comp = &((struct ionic_rxq_comp *)cq->base)[cq->tail_idx]; + else + comp = cq->base + + cq->desc_size * cq->tail_idx + + cq->desc_size - sizeof(*comp); if (!color_match(comp->pkt_type_color, cq->done_color)) return false; + dma_rmb(); + /* check for empty queue */ if (q->tail_idx == q->head_idx) return false; @@ -1185,7 +1188,6 @@ static void ionic_tx_clean(struct ionic_queue *q, bool in_napi) { struct ionic_tx_stats *stats = q_to_tx_stats(q); - struct ionic_qcq *qcq = q_to_qcq(q); struct sk_buff *skb; if (desc_info->xdpf) { @@ -1210,11 +1212,7 @@ static void ionic_tx_clean(struct ionic_queue *q, __le64 *cq_desc_hwstamp; u64 hwstamp; - cq_desc_hwstamp = - (void *)comp + - qcq->cq.desc_size - - sizeof(struct ionic_txq_comp) - - IONIC_HWSTAMP_CQ_NEGOFFSET; + cq_desc_hwstamp = (void *)comp - IONIC_HWSTAMP_CQ_NEGOFFSET; hwstamp = le64_to_cpu(*cq_desc_hwstamp); @@ -1249,11 +1247,18 @@ static bool ionic_tx_service(struct ionic_cq *cq, unsigned int pkts = 0; u16 index; - comp = &((struct ionic_txq_comp *)cq->base)[cq->tail_idx]; + if (likely(cq->desc_size == sizeof(*comp))) + comp = &((struct ionic_txq_comp *)cq->base)[cq->tail_idx]; + else + comp = cq->base + + cq->desc_size * cq->tail_idx + + cq->desc_size - sizeof(*comp); if (!color_match(comp->color, cq->done_color)) return false; + dma_rmb(); + /* clean the related q entries, there could be * several q entries completed for each cq completion */ diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_init.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_init.c index 9192c5ad5a1696..c6bdf1b6ac9d0a 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_init.c +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_init.c @@ -6,6 +6,7 @@ #include "qlcnic.h" #include "qlcnic_hw.h" +#include struct crb_addr_pair { u32 addr; @@ -740,159 +741,212 @@ qlcnic_has_mn(struct qlcnic_adapter *adapter) return 0; } -static -struct uni_table_desc *qlcnic_get_table_desc(const u8 *unirom, int section) -{ - u32 i, entries; - struct uni_table_desc *directory = (struct uni_table_desc *) &unirom[0]; - entries = le32_to_cpu(directory->num_entries); - - for (i = 0; i < entries; i++) { +#define FILEHEADER_SIZE (14 * 4) +#define QLCNIC_UNI_DIR_TYPE_OFF (8 * sizeof(__le32)) +#define QLCNIC_UNI_DIR_ENTRY_MIN_SIZE (9 * sizeof(__le32)) +#define QLCNIC_UNI_PRODUCT_ENTRY_MIN_SIZE \ + ((QLCNIC_UNI_FIRMWARE_IDX_OFF + 1) * sizeof(__le32)) +#define QLCNIC_UNI_VERSION_TAIL_SIZE 17 +#define QLCNIC_UNI_BOOTLD_SIZE \ + (QLCNIC_IMAGE_START - QLCNIC_BOOTLD_START) + +struct qlcnic_uni_data { + u32 offset; + u32 size; +}; - u32 offs = le32_to_cpu(directory->findex) + - i * le32_to_cpu(directory->entry_size); - u32 tab_type = le32_to_cpu(*((__le32 *)&unirom[offs] + 8)); +static bool qlcnic_rom_range_valid(size_t size, size_t offset, size_t len) +{ + return offset <= size && len <= size - offset; +} - if (tab_type == section) - return (struct uni_table_desc *) &unirom[offs]; - } +static bool qlcnic_rom_table_valid(size_t size, u32 offset, u32 entries, + u32 entry_size, u32 min_entry_size) +{ + if (entry_size < min_entry_size || offset > size) + return false; - return NULL; + return entries <= (size - offset) / entry_size; } -#define FILEHEADER_SIZE (14 * 4) - -static int -qlcnic_validate_header(struct qlcnic_adapter *adapter) +static int qlcnic_get_directory(struct qlcnic_adapter *adapter, + size_t *offset, u32 *entries, u32 *entry_size) { - const u8 *unirom = adapter->fw->data; - struct uni_table_desc *directory = (struct uni_table_desc *) &unirom[0]; - u32 entries, entry_size, tab_size, fw_file_size; - - fw_file_size = adapter->fw->size; + const struct firmware *fw = adapter->fw; + const u8 *directory = fw->data; - if (fw_file_size < FILEHEADER_SIZE) + if (fw->size < FILEHEADER_SIZE) return -EINVAL; - entries = le32_to_cpu(directory->num_entries); - entry_size = le32_to_cpu(directory->entry_size); - tab_size = le32_to_cpu(directory->findex) + (entries * entry_size); + *offset = get_unaligned_le32(directory + + offsetof(struct uni_table_desc, findex)); + *entries = get_unaligned_le32(directory + + offsetof(struct uni_table_desc, num_entries)); + *entry_size = get_unaligned_le32(directory + + offsetof(struct uni_table_desc, entry_size)); - if (fw_file_size < tab_size) + if (!qlcnic_rom_table_valid(fw->size, *offset, *entries, *entry_size, + QLCNIC_UNI_DIR_ENTRY_MIN_SIZE)) return -EINVAL; return 0; } -static int -qlcnic_validate_bootld(struct qlcnic_adapter *adapter) +static int qlcnic_get_table_desc(struct qlcnic_adapter *adapter, int section, + size_t *desc_offset) { - struct uni_table_desc *tab_desc; - struct uni_data_desc *descr; - u32 offs, tab_size, data_size, idx; const u8 *unirom = adapter->fw->data; - __le32 temp; - - temp = *((__le32 *)&unirom[adapter->file_prd_off] + - QLCNIC_UNI_BOOTLD_IDX_OFF); - idx = le32_to_cpu(temp); - tab_desc = qlcnic_get_table_desc(unirom, QLCNIC_UNI_DIR_SECT_BOOTLD); - - if (!tab_desc) - return -EINVAL; - - tab_size = le32_to_cpu(tab_desc->findex) + - le32_to_cpu(tab_desc->entry_size) * (idx + 1); - - if (adapter->fw->size < tab_size) - return -EINVAL; + size_t directory_offset; + u32 entries, entry_size; + size_t i; + int ret; - offs = le32_to_cpu(tab_desc->findex) + - le32_to_cpu(tab_desc->entry_size) * idx; - descr = (struct uni_data_desc *)&unirom[offs]; + ret = qlcnic_get_directory(adapter, &directory_offset, &entries, + &entry_size); + if (ret) + return ret; - data_size = le32_to_cpu(descr->findex) + le32_to_cpu(descr->size); + for (i = 0; i < entries; i++) { + size_t offset = directory_offset + i * entry_size; + u32 table_type; - if (adapter->fw->size < data_size) - return -EINVAL; + table_type = get_unaligned_le32(unirom + offset + + QLCNIC_UNI_DIR_TYPE_OFF); + if (table_type == section) { + *desc_offset = offset; + return 0; + } + } - return 0; + return -ENOENT; } static int -qlcnic_validate_fw(struct qlcnic_adapter *adapter) +qlcnic_validate_header(struct qlcnic_adapter *adapter) { - struct uni_table_desc *tab_desc; - struct uni_data_desc *descr; - const u8 *unirom = adapter->fw->data; - u32 offs, tab_size, data_size, idx; - __le32 temp; + u32 entries, entry_size; + size_t offset; + + return qlcnic_get_directory(adapter, &offset, &entries, &entry_size); +} - temp = *((__le32 *)&unirom[adapter->file_prd_off] + - QLCNIC_UNI_FIRMWARE_IDX_OFF); - idx = le32_to_cpu(temp); - tab_desc = qlcnic_get_table_desc(unirom, QLCNIC_UNI_DIR_SECT_FW); +static int qlcnic_get_data_desc(struct qlcnic_adapter *adapter, u32 section, + u32 index_offset, struct qlcnic_uni_data *data) +{ + size_t table_desc_offset, table_offset, desc_offset; + const struct firmware *fw = adapter->fw; + const u8 *unirom = fw->data; + size_t product_index_offset; + u32 entries, entry_size, idx; + int ret; - if (!tab_desc) + product_index_offset = adapter->file_prd_off + + (size_t)index_offset * sizeof(__le32); + if (!qlcnic_rom_range_valid(fw->size, product_index_offset, + sizeof(__le32))) return -EINVAL; - tab_size = le32_to_cpu(tab_desc->findex) + - le32_to_cpu(tab_desc->entry_size) * (idx + 1); + idx = get_unaligned_le32(unirom + product_index_offset); + ret = qlcnic_get_table_desc(adapter, section, &table_desc_offset); + if (ret) + return ret; - if (adapter->fw->size < tab_size) + table_offset = get_unaligned_le32(unirom + table_desc_offset + + offsetof(struct uni_table_desc, findex)); + entries = get_unaligned_le32(unirom + table_desc_offset + + offsetof(struct uni_table_desc, num_entries)); + entry_size = get_unaligned_le32(unirom + table_desc_offset + + offsetof(struct uni_table_desc, entry_size)); + if (!qlcnic_rom_table_valid(fw->size, table_offset, entries, + entry_size, sizeof(struct uni_data_desc)) || + idx >= entries) return -EINVAL; - offs = le32_to_cpu(tab_desc->findex) + - le32_to_cpu(tab_desc->entry_size) * idx; - descr = (struct uni_data_desc *)&unirom[offs]; - data_size = le32_to_cpu(descr->findex) + le32_to_cpu(descr->size); + desc_offset = table_offset + (size_t)idx * entry_size; + data->offset = get_unaligned_le32(unirom + desc_offset + + offsetof(struct uni_data_desc, findex)); + data->size = get_unaligned_le32(unirom + desc_offset + + offsetof(struct uni_data_desc, size)); - if (adapter->fw->size < data_size) + if (!qlcnic_rom_range_valid(fw->size, data->offset, data->size)) return -EINVAL; return 0; } +static int qlcnic_validate_bootld(struct qlcnic_adapter *adapter) +{ + struct qlcnic_uni_data data; + int ret; + + ret = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_BOOTLD, + QLCNIC_UNI_BOOTLD_IDX_OFF, &data); + if (ret) + return ret; + + return data.size < QLCNIC_UNI_BOOTLD_SIZE ? -EINVAL : 0; +} + +static int qlcnic_validate_fw(struct qlcnic_adapter *adapter) +{ + struct qlcnic_uni_data data; + int ret; + + ret = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_FW, + QLCNIC_UNI_FIRMWARE_IDX_OFF, &data); + if (ret) + return ret; + + return data.size < QLCNIC_UNI_VERSION_TAIL_SIZE ? -EINVAL : 0; +} + static int qlcnic_validate_product_offs(struct qlcnic_adapter *adapter) { - struct uni_table_desc *ptab_descr; + size_t table_desc_offset, table_offset; const u8 *unirom = adapter->fw->data; int mn_present = qlcnic_has_mn(adapter); - u32 entries, entry_size, tab_size, i; - __le32 temp; - - ptab_descr = qlcnic_get_table_desc(unirom, - QLCNIC_UNI_DIR_SECT_PRODUCT_TBL); - if (!ptab_descr) - return -EINVAL; + u32 entries, entry_size; + size_t i; + int ret; - entries = le32_to_cpu(ptab_descr->num_entries); - entry_size = le32_to_cpu(ptab_descr->entry_size); - tab_size = le32_to_cpu(ptab_descr->findex) + (entries * entry_size); + ret = qlcnic_get_table_desc(adapter, QLCNIC_UNI_DIR_SECT_PRODUCT_TBL, + &table_desc_offset); + if (ret) + return ret; - if (adapter->fw->size < tab_size) + table_offset = get_unaligned_le32(unirom + table_desc_offset + + offsetof(struct uni_table_desc, findex)); + entries = get_unaligned_le32(unirom + table_desc_offset + + offsetof(struct uni_table_desc, num_entries)); + entry_size = get_unaligned_le32(unirom + table_desc_offset + + offsetof(struct uni_table_desc, entry_size)); + if (!qlcnic_rom_table_valid(adapter->fw->size, table_offset, entries, + entry_size, + QLCNIC_UNI_PRODUCT_ENTRY_MIN_SIZE)) return -EINVAL; nomn: for (i = 0; i < entries; i++) { - - u32 flags, file_chiprev, offs; + size_t offset = table_offset + i * entry_size; u8 chiprev = adapter->ahw->revision_id; + u32 flags, file_chiprev; u32 flagbit; - offs = le32_to_cpu(ptab_descr->findex) + - i * le32_to_cpu(ptab_descr->entry_size); - temp = *((__le32 *)&unirom[offs] + QLCNIC_UNI_FLAGS_OFF); - flags = le32_to_cpu(temp); - temp = *((__le32 *)&unirom[offs] + QLCNIC_UNI_CHIP_REV_OFF); - file_chiprev = le32_to_cpu(temp); + flags = get_unaligned_le32(unirom + offset + + QLCNIC_UNI_FLAGS_OFF * sizeof(__le32)); + file_chiprev = get_unaligned_le32(unirom + offset + + QLCNIC_UNI_CHIP_REV_OFF * + sizeof(__le32)); flagbit = mn_present ? 1 : 2; if ((chiprev == file_chiprev) && ((1ULL << flagbit) & flags)) { - adapter->file_prd_off = offs; + if (offset > U32_MAX) + return -EINVAL; + + adapter->file_prd_off = offset; return 0; } } @@ -933,93 +987,81 @@ qlcnic_validate_unified_romimage(struct qlcnic_adapter *adapter) return 0; } -static -struct uni_data_desc *qlcnic_get_data_desc(struct qlcnic_adapter *adapter, - u32 section, u32 idx_offset) -{ - const u8 *unirom = adapter->fw->data; - struct uni_table_desc *tab_desc; - u32 offs, idx; - __le32 temp; - - temp = *((__le32 *)&unirom[adapter->file_prd_off] + idx_offset); - idx = le32_to_cpu(temp); - - tab_desc = qlcnic_get_table_desc(unirom, section); - - if (tab_desc == NULL) - return NULL; - - offs = le32_to_cpu(tab_desc->findex) + - le32_to_cpu(tab_desc->entry_size) * idx; - - return (struct uni_data_desc *)&unirom[offs]; -} - -static u8 * -qlcnic_get_bootld_offs(struct qlcnic_adapter *adapter) +static int qlcnic_get_bootld_data(struct qlcnic_adapter *adapter, + const u8 **bootld) { u32 offs = QLCNIC_BOOTLD_START; - struct uni_data_desc *data_desc; - - data_desc = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_BOOTLD, - QLCNIC_UNI_BOOTLD_IDX_OFF); + struct qlcnic_uni_data data; + int ret; - if (adapter->ahw->fw_type == QLCNIC_UNIFIED_ROMIMAGE) - offs = le32_to_cpu(data_desc->findex); + if (adapter->ahw->fw_type == QLCNIC_UNIFIED_ROMIMAGE) { + ret = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_BOOTLD, + QLCNIC_UNI_BOOTLD_IDX_OFF, &data); + if (ret || data.size < QLCNIC_UNI_BOOTLD_SIZE) + return -EINVAL; + offs = data.offset; + } else if (!qlcnic_rom_range_valid(adapter->fw->size, offs, + QLCNIC_UNI_BOOTLD_SIZE)) { + return -EINVAL; + } - return (u8 *)&adapter->fw->data[offs]; + *bootld = adapter->fw->data + offs; + return 0; } -static u8 * -qlcnic_get_fw_offs(struct qlcnic_adapter *adapter) +static int qlcnic_get_fw_data(struct qlcnic_adapter *adapter, + const u8 **image, u32 *image_size) { u32 offs = QLCNIC_IMAGE_START; - struct uni_data_desc *data_desc; - - data_desc = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_FW, - QLCNIC_UNI_FIRMWARE_IDX_OFF); - if (adapter->ahw->fw_type == QLCNIC_UNIFIED_ROMIMAGE) - offs = le32_to_cpu(data_desc->findex); - - return (u8 *)&adapter->fw->data[offs]; -} + struct qlcnic_uni_data data; + int ret; -static u32 qlcnic_get_fw_size(struct qlcnic_adapter *adapter) -{ - struct uni_data_desc *data_desc; - const u8 *unirom = adapter->fw->data; + if (adapter->ahw->fw_type == QLCNIC_UNIFIED_ROMIMAGE) { + ret = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_FW, + QLCNIC_UNI_FIRMWARE_IDX_OFF, &data); + if (ret) + return ret; + offs = data.offset; + *image_size = data.size; + } else { + if (!qlcnic_rom_range_valid(adapter->fw->size, + QLCNIC_FW_SIZE_OFFSET, + sizeof(__le32))) + return -EINVAL; + *image_size = get_unaligned_le32(adapter->fw->data + + QLCNIC_FW_SIZE_OFFSET); + } - data_desc = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_FW, - QLCNIC_UNI_FIRMWARE_IDX_OFF); + if (!qlcnic_rom_range_valid(adapter->fw->size, offs, *image_size)) + return -EINVAL; - if (adapter->ahw->fw_type == QLCNIC_UNIFIED_ROMIMAGE) - return le32_to_cpu(data_desc->size); - else - return le32_to_cpu(*(__le32 *)&unirom[QLCNIC_FW_SIZE_OFFSET]); + *image = adapter->fw->data + offs; + return 0; } static u32 qlcnic_get_fw_version(struct qlcnic_adapter *adapter) { - struct uni_data_desc *fw_data_desc; + char ver_str[QLCNIC_UNI_VERSION_TAIL_SIZE + 1]; const struct firmware *fw = adapter->fw; + struct qlcnic_uni_data data; u32 major, minor, sub; - __le32 version_offset; - const u8 *ver_str; int i, ret; - if (adapter->ahw->fw_type != QLCNIC_UNIFIED_ROMIMAGE) { - version_offset = *(__le32 *)&fw->data[QLCNIC_FW_VERSION_OFFSET]; - return le32_to_cpu(version_offset); - } + if (adapter->ahw->fw_type != QLCNIC_UNIFIED_ROMIMAGE) + return get_unaligned_le32(fw->data + QLCNIC_FW_VERSION_OFFSET); + + ret = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_FW, + QLCNIC_UNI_FIRMWARE_IDX_OFF, &data); + if (ret || data.size < QLCNIC_UNI_VERSION_TAIL_SIZE) + return 0; - fw_data_desc = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_FW, - QLCNIC_UNI_FIRMWARE_IDX_OFF); - ver_str = fw->data + le32_to_cpu(fw_data_desc->findex) + - le32_to_cpu(fw_data_desc->size) - 17; + memcpy(ver_str, fw->data + data.offset + data.size - + QLCNIC_UNI_VERSION_TAIL_SIZE, + QLCNIC_UNI_VERSION_TAIL_SIZE); + ver_str[QLCNIC_UNI_VERSION_TAIL_SIZE] = '\0'; for (i = 0; i < 12; i++) { - if (!strncmp(&ver_str[i], "REV=", 4)) { + if (!strncmp(ver_str + i, "REV=", 4)) { ret = sscanf(&ver_str[i+4], "%u.%u.%u ", &major, &minor, &sub); if (ret != 3) @@ -1034,18 +1076,15 @@ static u32 qlcnic_get_fw_version(struct qlcnic_adapter *adapter) static u32 qlcnic_get_bios_version(struct qlcnic_adapter *adapter) { - const struct firmware *fw = adapter->fw; u32 bios_ver, prd_off = adapter->file_prd_off; - u8 *version_offset; - __le32 temp; + const struct firmware *fw = adapter->fw; - if (adapter->ahw->fw_type != QLCNIC_UNIFIED_ROMIMAGE) { - version_offset = (u8 *)&fw->data[QLCNIC_BIOS_VERSION_OFFSET]; - return le32_to_cpu(*(__le32 *)version_offset); - } + if (adapter->ahw->fw_type != QLCNIC_UNIFIED_ROMIMAGE) + return get_unaligned_le32(fw->data + QLCNIC_BIOS_VERSION_OFFSET); - temp = *((__le32 *)(&fw->data[prd_off]) + QLCNIC_UNI_BIOS_VERSION_OFF); - bios_ver = le32_to_cpu(temp); + bios_ver = get_unaligned_le32(fw->data + prd_off + + QLCNIC_UNI_BIOS_VERSION_OFF * + sizeof(__le32)); return (bios_ver << 16) + ((bios_ver >> 8) & 0xff00) + (bios_ver >> 24); } @@ -1106,24 +1145,26 @@ static const char *fw_name[] = { int qlcnic_load_firmware(struct qlcnic_adapter *adapter) { - __le64 *ptr64; - u32 i, flashaddr, size; const struct firmware *fw = adapter->fw; struct pci_dev *pdev = adapter->pdev; + const u8 *bootld, *image; + u32 i, flashaddr, image_size; + int ret; dev_info(&pdev->dev, "loading firmware from %s\n", fw_name[adapter->ahw->fw_type]); if (fw) { + u32 words, remainder; u64 data; - size = (QLCNIC_IMAGE_START - QLCNIC_BOOTLD_START) / 8; - - ptr64 = (__le64 *)qlcnic_get_bootld_offs(adapter); + ret = qlcnic_get_bootld_data(adapter, &bootld); + if (ret) + return ret; flashaddr = QLCNIC_BOOTLD_START; - for (i = 0; i < size; i++) { - data = le64_to_cpu(ptr64[i]); + for (i = 0; i < QLCNIC_UNI_BOOTLD_SIZE / sizeof(u64); i++) { + data = get_unaligned_le64(bootld + i * sizeof(u64)); if (qlcnic_pci_mem_write_2M(adapter, flashaddr, data)) return -EIO; @@ -1131,13 +1172,15 @@ qlcnic_load_firmware(struct qlcnic_adapter *adapter) flashaddr += 8; } - size = qlcnic_get_fw_size(adapter) / 8; - - ptr64 = (__le64 *)qlcnic_get_fw_offs(adapter); + ret = qlcnic_get_fw_data(adapter, &image, &image_size); + if (ret) + return ret; + words = image_size / sizeof(u64); + remainder = image_size % sizeof(u64); flashaddr = QLCNIC_IMAGE_START; - for (i = 0; i < size; i++) { - data = le64_to_cpu(ptr64[i]); + for (i = 0; i < words; i++) { + data = get_unaligned_le64(image + i * sizeof(u64)); if (qlcnic_pci_mem_write_2M(adapter, flashaddr, data)) @@ -1146,9 +1189,11 @@ qlcnic_load_firmware(struct qlcnic_adapter *adapter) flashaddr += 8; } - size = qlcnic_get_fw_size(adapter) % 8; - if (size) { - data = le64_to_cpu(ptr64[i]); + if (remainder) { + __le64 tail = 0; + + memcpy(&tail, image + words * sizeof(u64), remainder); + data = le64_to_cpu(tail); if (qlcnic_pci_mem_write_2M(adapter, flashaddr, data)) @@ -1156,10 +1201,9 @@ qlcnic_load_firmware(struct qlcnic_adapter *adapter) } } else { - u64 data; - u32 hi, lo; - int ret; struct qlcnic_flt_entry bootld_entry; + u32 hi, lo, size; + u64 data; ret = qlcnic_get_flt_entry(adapter, QLCNIC_BOOTLD_REGION, &bootld_entry); @@ -1200,26 +1244,33 @@ qlcnic_load_firmware(struct qlcnic_adapter *adapter) static int qlcnic_validate_firmware(struct qlcnic_adapter *adapter) { - u32 val; - u32 ver, bios, min_size; - struct pci_dev *pdev = adapter->pdev; const struct firmware *fw = adapter->fw; + struct pci_dev *pdev = adapter->pdev; u8 fw_type = adapter->ahw->fw_type; + u32 ver, bios, min_size; + const u8 *data; + u32 data_size; + u32 val; + + if (fw_type == QLCNIC_UNIFIED_ROMIMAGE) + min_size = QLCNIC_UNI_FW_MIN_SIZE; + else + min_size = QLCNIC_FW_MIN_SIZE; + + if (fw->size < min_size) + return -EINVAL; if (fw_type == QLCNIC_UNIFIED_ROMIMAGE) { if (qlcnic_validate_unified_romimage(adapter)) return -EINVAL; - - min_size = QLCNIC_UNI_FW_MIN_SIZE; } else { - val = le32_to_cpu(*(__le32 *)&fw->data[QLCNIC_FW_MAGIC_OFFSET]); + val = get_unaligned_le32(fw->data + QLCNIC_FW_MAGIC_OFFSET); if (val != QLCNIC_BDINFO_MAGIC) return -EINVAL; - - min_size = QLCNIC_FW_MIN_SIZE; } - if (fw->size < min_size) + if (qlcnic_get_bootld_data(adapter, &data) || + qlcnic_get_fw_data(adapter, &data, &data_size)) return -EINVAL; val = qlcnic_get_fw_version(adapter); diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c index 305ae15ae8f393..e6f48dc9fe64d7 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c @@ -394,6 +394,7 @@ struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb, if (!skbn) return NULL; + skbn->dev = skb->dev; skb_reserve(skbn, RMNET_MAP_DEAGGR_HEADROOM); skb_put(skbn, packet_len); memcpy(skbn->data, skb->data, packet_len); diff --git a/drivers/net/ethernet/realtek/8139cp.c b/drivers/net/ethernet/realtek/8139cp.c index 5652da8a178c08..9016527e229ac0 100644 --- a/drivers/net/ethernet/realtek/8139cp.c +++ b/drivers/net/ethernet/realtek/8139cp.c @@ -2066,7 +2066,7 @@ static int __maybe_unused cp_suspend(struct device *device) /* Disable Rx and Tx */ cpw16 (IntrMask, 0); - cpw8 (Cmd, cpr8 (Cmd) & (~RxOn | ~TxOn)); + cpw8 (Cmd, cpr8 (Cmd) & ~(RxOn | TxOn)); spin_unlock_irqrestore (&cp->lock, flags); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c index caac85fc08f1b6..d4ace392489153 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c @@ -194,7 +194,7 @@ static void dwmac1000_set_filter(struct mac_device_info *hw, dwmac1000_set_mchash(ioaddr, mc_filter, mcbitslog2); /* Handle multiple unicast addresses (perfect filtering) */ - if (netdev_uc_count(dev) > perfect_addr_number) + if (netdev_uc_count(dev) + 1 > perfect_addr_number) /* Switch to promiscuous mode if more than unicast * addresses are requested than supported by hardware. */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c index c6fcfae27c3d0e..18b357b257cc09 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c @@ -521,7 +521,7 @@ static void dwmac4_set_filter(struct mac_device_info *hw, value |= GMAC_PACKET_FILTER_HPF; /* Handle multiple unicast addresses */ - if (netdev_uc_count(dev) > hw->unicast_filter_entries) { + if (netdev_uc_count(dev) + 1 > hw->unicast_filter_entries) { /* Switch to promiscuous mode if more than 128 addrs * are required */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c index 52054f31376d50..fc6ddb51c6820f 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c @@ -532,7 +532,7 @@ static void dwxgmac2_set_filter(struct mac_device_info *hw, dwxgmac2_set_mchash(ioaddr, mc_filter, mcbitslog2); /* Handle multiple unicast addresses */ - if (netdev_uc_count(dev) > hw->unicast_filter_entries) { + if (netdev_uc_count(dev) + 1 > hw->unicast_filter_entries) { value |= XGMAC_FILTER_PR; } else { struct netdev_hw_addr *ha; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h index 8ba8f03e1ce037..7582fca637419d 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h @@ -265,8 +265,6 @@ struct stmmac_priv { u32 rx_coal_frames[MTL_MAX_RX_QUEUES]; int hwts_tx_en; - /* skb_shinfo(skb)->gso_type types that we handle */ - unsigned int gso_enabled_types; bool tx_path_in_lpi_mode; bool sph_active; bool sph_capable; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index a71f0df263785d..3a437409f78aef 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -1330,6 +1330,10 @@ static int stmmac_init_phy(struct net_device *dev) struct phy_device *phydev; if (addr < 0) { + /* If a custom PCS is in use, no PHY is needed */ + if (priv->hw->phylink_pcs) + return 0; + netdev_err(priv->dev, "no phy found\n"); return -ENODEV; } @@ -1527,9 +1531,9 @@ static void stmmac_display_rings(struct stmmac_priv *priv, static unsigned int stmmac_rx_offset(struct stmmac_priv *priv) { if (stmmac_xdp_is_enabled(priv)) - return XDP_PACKET_HEADROOM; + return XDP_PACKET_HEADROOM + NET_IP_ALIGN; - return NET_SKB_PAD; + return NET_SKB_PAD + NET_IP_ALIGN; } static int stmmac_set_bfsize(int mtu) @@ -2719,7 +2723,8 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) tx_desc = stmmac_get_tx_desc(priv, tx_q, entry); dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr); - meta = xsk_buff_get_metadata(pool, xdp_desc.addr); + meta = xsk_buff_get_metadata(pool, xdp_desc.addr, + xdp_desc.options); xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len); /* To return XDP buffer to XSK pool, we simple call @@ -4371,18 +4376,6 @@ static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) stmmac_set_queue_tx_tail_ptr(priv, tx_q, queue, tx_q->cur_tx); } -static void stmmac_set_gso_types(struct stmmac_priv *priv, bool tso) -{ - if (!tso) { - priv->gso_enabled_types = 0; - } else { - /* Manage oversized TCP frames for GMAC4 device */ - priv->gso_enabled_types = SKB_GSO_TCPV4 | SKB_GSO_TCPV6; - if (priv->plat->core_type == DWMAC_CORE_GMAC4) - priv->gso_enabled_types |= SKB_GSO_UDP_L4; - } -} - static void stmmac_set_gso_features(struct net_device *ndev) { struct stmmac_priv *priv = netdev_priv(ndev); @@ -4416,8 +4409,6 @@ static void stmmac_set_gso_features(struct net_device *ndev) if (priv->plat->core_type == DWMAC_CORE_GMAC4) ndev->hw_features |= NETIF_F_GSO_UDP_L4; - stmmac_set_gso_types(priv, true); - dev_info(priv->device, "TSO feature enabled\n"); } @@ -4765,8 +4756,7 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) stmmac_stop_sw_lpi(priv); - if (skb_is_gso(skb) && - skb_shinfo(skb)->gso_type & priv->gso_enabled_types) + if (skb_is_gso(skb)) return stmmac_tso_xmit(skb, dev); if (priv->est && priv->est->enable && @@ -6198,8 +6188,6 @@ static int stmmac_set_features(struct net_device *netdev, stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); } - stmmac_set_gso_types(priv, features & NETIF_F_TSO); - if (features & NETIF_F_HW_VLAN_CTAG_RX) priv->hw->hw_vlan_en = true; else diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c index a0c75886587c35..f97f32369e903f 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c @@ -415,11 +415,17 @@ static int stmmac_test_mmc(struct stmmac_priv *priv) stmmac_mmc_read(priv, priv->mmcaddr, &final); /* - * The number of MMC counters available depends on HW configuration - * so we just use this one to validate the feature. I hope there is - * not a version without this counter. + * The number of MMC counters available depends on HW configuration, + * and there doesn't seem to be a way to enumerate the implemented + * counters. + * + * Let's check a hand-picked set of counters, knowing that : + * - Starfive JH7110 doesn't implement mmc_tx_framecount_g + * - Amlogic SM1 doesn't implement any mmc_tx_* + * */ - if (final.mmc_tx_framecount_g <= initial.mmc_tx_framecount_g) + if (final.mmc_tx_framecount_g <= initial.mmc_tx_framecount_g && + final.mmc_rx_framecount_gb <= initial.mmc_rx_framecount_gb) return -EINVAL; return 0; @@ -493,6 +499,21 @@ static int stmmac_filter_check(struct stmmac_priv *priv) return -EOPNOTSUPP; } +static int stmmac_uc_filter_check(struct stmmac_priv *priv) +{ + /* For tests involving the UC filter, we need at least one empty + * slot in the UC filter. The UC filters contains netdev_uc_count() + 1 + * entries: The dev->uc list + one entry for the HW address. + * + * Having an empty slot therefore means netdev_uc_count() + 2 entries + * can fit in the filter + */ + if (netdev_uc_count(priv->dev) + 2 > priv->hw->unicast_filter_entries) + return -EOPNOTSUPP; + + return 0; +} + static bool stmmac_hash_check(struct stmmac_priv *priv, unsigned char *addr) { int mc_offset = 32 - priv->hw->mcast_bits_log2; @@ -584,7 +605,7 @@ static int stmmac_test_pfilt(struct stmmac_priv *priv) if (stmmac_filter_check(priv)) return -EOPNOTSUPP; - if (netdev_uc_count(priv->dev) >= priv->hw->unicast_filter_entries) + if (stmmac_uc_filter_check(priv)) return -EOPNOTSUPP; while (--tries) { @@ -628,7 +649,7 @@ static int stmmac_test_mcfilt(struct stmmac_priv *priv) if (stmmac_filter_check(priv)) return -EOPNOTSUPP; - if (netdev_uc_count(priv->dev) >= priv->hw->unicast_filter_entries) + if (stmmac_uc_filter_check(priv)) return -EOPNOTSUPP; if (netdev_mc_count(priv->dev) >= priv->hw->multicast_filter_bins) return -EOPNOTSUPP; @@ -674,7 +695,7 @@ static int stmmac_test_ucfilt(struct stmmac_priv *priv) if (stmmac_filter_check(priv)) return -EOPNOTSUPP; - if (netdev_uc_count(priv->dev) >= priv->hw->unicast_filter_entries) + if (stmmac_uc_filter_check(priv)) return -EOPNOTSUPP; if (netdev_mc_count(priv->dev) >= priv->hw->multicast_filter_bins) return -EOPNOTSUPP; @@ -738,12 +759,24 @@ static int stmmac_test_flowctrl(struct stmmac_priv *priv) struct phy_device *phydev = priv->dev->phydev; u32 rx_cnt = priv->plat->rx_queues_to_use; struct stmmac_test_priv *tpriv; + unsigned int rx_fifo_size; unsigned int pkt_count; int i, ret = 0; if (!phydev || (!phydev->pause && !phydev->asym_pause)) return -EOPNOTSUPP; + rx_fifo_size = priv->plat->rx_fifo_size; + if (!rx_fifo_size) + rx_fifo_size = priv->dma_cap.rx_fifo_size; + + /* No pause frame is emitted if we don't have at least 4096 bytes per + * queue, except on dwmac100. + */ + if (priv->plat->core_type != DWMAC_CORE_MAC100 && + rx_fifo_size / priv->plat->rx_queues_to_use < 4096) + return -EOPNOTSUPP; + tpriv = kzalloc_obj(*tpriv); if (!tpriv) return -ENOMEM; @@ -757,9 +790,7 @@ static int stmmac_test_flowctrl(struct stmmac_priv *priv) dev_add_pack(&tpriv->pt); /* Compute minimum number of packets to make FIFO full */ - pkt_count = priv->plat->rx_fifo_size; - if (!pkt_count) - pkt_count = priv->dma_cap.rx_fifo_size; + pkt_count = rx_fifo_size; pkt_count /= 1400; pkt_count *= 2; @@ -1452,11 +1483,11 @@ static int __stmmac_test_l4filt(struct stmmac_priv *priv, u32 dst, u32 src, struct { struct flow_dissector_key_basic bkey; struct flow_dissector_key_ports key; - } __aligned(BITS_PER_LONG / 8) keys; + } __aligned(BITS_PER_LONG / 8) keys = { }; struct { struct flow_dissector_key_basic bmask; struct flow_dissector_key_ports mask; - } __aligned(BITS_PER_LONG / 8) masks; + } __aligned(BITS_PER_LONG / 8) masks = { }; unsigned long dummy_cookie = 0xdeadbeef; struct stmmac_packet_attrs attr = { }; struct flow_dissector *dissector; @@ -1509,6 +1540,8 @@ static int __stmmac_test_l4filt(struct stmmac_priv *priv, u32 dst, u32 src, keys.bkey.ip_proto = udp ? IPPROTO_UDP : IPPROTO_TCP; keys.key.src = htons(src); keys.key.dst = htons(dst); + /* Match the full IP proto field */ + masks.bmask.ip_proto = 0xff; masks.mask.src = src_mask; masks.mask.dst = dst_mask; diff --git a/drivers/net/ethernet/wangxun/libwx/wx_ptp.c b/drivers/net/ethernet/wangxun/libwx/wx_ptp.c index 44f3e65052468f..1165518d55225c 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_ptp.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_ptp.c @@ -555,13 +555,11 @@ static int wx_ptp_set_timestamp_mode(struct wx *wx, { u32 tsync_tx_ctl = WX_TSC_1588_CTL_ENABLED; u32 tsync_rx_ctl = WX_PSR_1588_CTL_ENABLED; - DECLARE_BITMAP(flags, WX_PF_FLAGS_NBITS); u32 tsync_rx_mtrl = PTP_EV_PORT << 16; + bool rx_tstamp = false; bool is_l2 = false; u32 regval; - memcpy(flags, wx->flags, sizeof(wx->flags)); - switch (config->tx_type) { case HWTSTAMP_TX_OFF: tsync_tx_ctl = 0; @@ -576,20 +574,16 @@ static int wx_ptp_set_timestamp_mode(struct wx *wx, case HWTSTAMP_FILTER_NONE: tsync_rx_ctl = 0; tsync_rx_mtrl = 0; - clear_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, flags); - clear_bit(WX_FLAG_RX_HWTSTAMP_IN_REGISTER, flags); break; case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: tsync_rx_ctl |= WX_PSR_1588_CTL_TYPE_L4_V1; tsync_rx_mtrl |= WX_PSR_1588_MSG_V1_SYNC; - set_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, flags); - set_bit(WX_FLAG_RX_HWTSTAMP_IN_REGISTER, flags); + rx_tstamp = true; break; case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: tsync_rx_ctl |= WX_PSR_1588_CTL_TYPE_L4_V1; tsync_rx_mtrl |= WX_PSR_1588_MSG_V1_DELAY_REQ; - set_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, flags); - set_bit(WX_FLAG_RX_HWTSTAMP_IN_REGISTER, flags); + rx_tstamp = true; break; case HWTSTAMP_FILTER_PTP_V2_EVENT: case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: @@ -602,9 +596,8 @@ static int wx_ptp_set_timestamp_mode(struct wx *wx, case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: tsync_rx_ctl |= WX_PSR_1588_CTL_TYPE_EVENT_V2; is_l2 = true; + rx_tstamp = true; config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; - set_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, flags); - set_bit(WX_FLAG_RX_HWTSTAMP_IN_REGISTER, flags); break; default: /* register PSR_1588_MSG must be set in order to do V1 packets, @@ -643,7 +636,8 @@ static int wx_ptp_set_timestamp_mode(struct wx *wx, WX_WRITE_FLUSH(wx); /* configure adapter flags only when HW is actually configured */ - memcpy(wx->flags, flags, sizeof(wx->flags)); + assign_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, wx->flags, rx_tstamp); + assign_bit(WX_FLAG_RX_HWTSTAMP_IN_REGISTER, wx->flags, rx_tstamp); /* clear TX/RX timestamp state, just to be sure */ wx_ptp_clear_tx_timestamp(wx); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_type.h b/drivers/net/ethernet/wangxun/libwx/wx_type.h index 65e3e55db1cffb..0520288d18ab97 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_type.h +++ b/drivers/net/ethernet/wangxun/libwx/wx_type.h @@ -1427,7 +1427,7 @@ struct wx { }; #define WX_INTR_ALL (~0ULL) -#define WX_INTR_Q(i) BIT((i)) +#define WX_INTR_Q(i) BIT_ULL((i)) /* register operations */ #define wr32(a, reg, value) writel((value), ((a)->hw_addr + (reg))) diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_irq.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_irq.c index 8746318ad3bcb8..5ad1ff7c3ce975 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_irq.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_irq.c @@ -164,6 +164,7 @@ static irqreturn_t txgbe_misc_irq_thread_fn(int irq, void *data) struct wx *wx = txgbe->wx; unsigned int nhandled = 0; unsigned int sub_irq; + u64 misc_mask; u32 eicr; eicr = txgbe->eicr; @@ -183,7 +184,9 @@ static irqreturn_t txgbe_misc_irq_thread_fn(int irq, void *data) nhandled++; } - wx_intr_enable(wx, TXGBE_INTR_MISC(wx)); + misc_mask = wx->pdev->msix_enabled ? TXGBE_INTR_MISC(wx) : BIT(0); + if (!test_bit(WX_STATE_DOWN, wx->state)) + wx_intr_enable(wx, misc_mask); return (nhandled > 0 ? IRQ_HANDLED : IRQ_NONE); } diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h b/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h index 877234e3fdc2b6..fddcb011fa2fd3 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h @@ -303,7 +303,7 @@ struct txgbe_fdir_filter { #define TXGBE_DEFAULT_RX_WORK 128 #endif -#define TXGBE_INTR_MISC(A) BIT((A)->num_q_vectors) +#define TXGBE_INTR_MISC(A) BIT_ULL((A)->num_q_vectors) #define TXGBE_INTR_QALL(A) (TXGBE_INTR_MISC(A) - 1) #define TXGBE_MAX_EITR GENMASK(11, 3) diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c index 2f6e77cce385fa..298efc76a56b0c 100644 --- a/drivers/net/gtp.c +++ b/drivers/net/gtp.c @@ -1549,6 +1549,8 @@ static int gtp_newlink(struct net_device *dev, out_encap: gtp_encap_disable(gtp); out_hashtable: + /* Wait for RCU readers that may still reference this gtp_dev. */ + synchronize_net(); kfree(gtp->addr_hash); kfree(gtp->tid_hash); return err; diff --git a/drivers/net/ipa/ipa_main.c b/drivers/net/ipa/ipa_main.c index 788dd99af2a4dc..8d82fd5bb24b97 100644 --- a/drivers/net/ipa/ipa_main.c +++ b/drivers/net/ipa/ipa_main.c @@ -969,12 +969,12 @@ static void ipa_remove(struct platform_device *pdev) } if (ret) { /* - * Not cleaning up here properly might also yield a - * crash later on. As the device is still unregistered - * in this case, this might even yield a crash later on. + * Continuing teardown after failing to stop the modem + * could crash, so leave the remaining resources allocated. */ dev_err(dev, "Failed to stop modem (%pe), leaking resources\n", ERR_PTR(ret)); + pm_runtime_put_noidle(dev); return; } diff --git a/drivers/net/netdevsim/netdev.c b/drivers/net/netdevsim/netdev.c index 4e9d7e10b527e5..b4a99f3ceac60b 100644 --- a/drivers/net/netdevsim/netdev.c +++ b/drivers/net/netdevsim/netdev.c @@ -529,6 +529,7 @@ static void nsim_del_napi(struct netdevsim *ns) for (i = 0; i < dev->num_rx_queues; i++) { struct nsim_rq *rq = ns->rq[i]; + netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, NULL); napi_disable_locked(&rq->napi); __netif_napi_del_locked(&rq->napi); } @@ -826,6 +827,7 @@ nsim_queue_start(struct net_device *dev, struct netdev_queue_config *qcfg, } ns->rq[idx] = qmem->rq; + netif_queue_set_napi(dev, idx, NETDEV_QUEUE_TYPE_RX, &ns->rq[idx]->napi); napi_enable_locked(&ns->rq[idx]->napi); return 0; diff --git a/drivers/net/phy/air_en8811h.c b/drivers/net/phy/air_en8811h.c index edd49c193e4768..0eeb7b9a4e260b 100644 --- a/drivers/net/phy/air_en8811h.c +++ b/drivers/net/phy/air_en8811h.c @@ -1173,13 +1173,6 @@ static int en8811h_probe(struct phy_device *phydev) if (ret) return ret; - /* Configure led gpio pins as output */ - ret = air_phy_buckpbus_reg_modify(phydev, EN8811H_GPIO_OUTPUT, - EN8811H_GPIO_OUTPUT_345, - EN8811H_GPIO_OUTPUT_345); - if (ret < 0) - return ret; - return 0; } @@ -1324,6 +1317,17 @@ static int en8811h_config_init(struct phy_device *phydev) return ret; } + /* Configure led gpio pins as output. Must be redone on every + * .config_init(), not just once in .probe(): en8811h_restart_mcu() + * resets buckpbus-mapped MCU state (incl. this register) on every + * call after the first, e.g. on link renegotiation or ifup/ifdown. + */ + ret = air_phy_buckpbus_reg_modify(phydev, EN8811H_GPIO_OUTPUT, + EN8811H_GPIO_OUTPUT_345, + EN8811H_GPIO_OUTPUT_345); + if (ret < 0) + return ret; + return 0; } diff --git a/drivers/net/ppp/pppox.c b/drivers/net/ppp/pppox.c index 5861a2f6ce3e69..a6f72c813bef95 100644 --- a/drivers/net/ppp/pppox.c +++ b/drivers/net/ppp/pppox.c @@ -74,7 +74,9 @@ int pppox_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) switch (cmd) { case PPPIOCGCHAN: { + struct sk_buff *skb; int index; + rc = -ENOTCONN; if (!(sk->sk_state & PPPOX_CONNECTED)) break; @@ -85,7 +87,22 @@ int pppox_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) break; rc = 0; + /* PPPIOCGCHAN historically marks the userspace handoff to + * generic PPP; pppd then attaches the returned channel to + * /dev/ppp. + */ sk->sk_state |= PPPOX_BOUND; + /* Let lockless receive paths finish queueing against the old + * state. + */ + synchronize_net(); + /* Drain packets queued before the handoff because a bound + * socket is no longer readable. + */ + while ((skb = skb_dequeue(&sk->sk_receive_queue))) { + skb_orphan(skb); + ppp_input(&po->chan, skb); + } break; } default: diff --git a/drivers/net/slip/slip.c b/drivers/net/slip/slip.c index 85b2438e892351..fdd6f63fb849e3 100644 --- a/drivers/net/slip/slip.c +++ b/drivers/net/slip/slip.c @@ -886,8 +886,6 @@ err_exit: * Close down a SLIP channel. * This means flushing out any pending queues, and then returning. This * call is serialized against other ldisc functions. - * - * We also use this method fo a hangup event */ static void slip_close(struct tty_struct *tty) @@ -915,11 +913,6 @@ static void slip_close(struct tty_struct *tty) unregister_netdev(sl->dev); /* sl_uninit() has dropped the slip_devs[] entry by now */ } - -static void slip_hangup(struct tty_struct *tty) -{ - slip_close(tty); -} /************************************************************************ * STANDARD SLIP ENCAPSULATION * ************************************************************************/ @@ -1280,7 +1273,6 @@ static struct tty_ldisc_ops sl_ldisc = { .name = "slip", .open = slip_open, .close = slip_close, - .hangup = slip_hangup, .ioctl = slip_ioctl, .receive_buf = slip_receive_buf, .write_wakeup = slip_write_wakeup, diff --git a/drivers/net/thunderbolt/main.c b/drivers/net/thunderbolt/main.c index 2a1728621887aa..82cef20092b8cb 100644 --- a/drivers/net/thunderbolt/main.c +++ b/drivers/net/thunderbolt/main.c @@ -904,9 +904,11 @@ static int tbnet_poll(struct napi_struct *napi, int budget) le32_to_cpu(net->rx_hdr.frame_count) - 1; rx_packets++; - net->stats.rx_bytes += frame_size; if (last) { + /* Before eth_type_trans() pulls the Ethernet header. */ + net->stats.rx_packets++; + net->stats.rx_bytes += skb->len; skb->protocol = eth_type_trans(skb, net->dev); trace_tbnet_rx_skb(skb); napi_gro_receive(&net->napi, skb); @@ -914,8 +916,6 @@ static int tbnet_poll(struct napi_struct *napi, int budget) } } - net->stats.rx_packets += rx_packets; - if (cleaned_count) tbnet_alloc_rx_buffers(net, cleaned_count); diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 3e2a5876c6c8ce..e34c52d059d398 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -3444,17 +3444,31 @@ static void virtnet_rx_resume_all(struct virtnet_info *vi) static int virtnet_rx_resize(struct virtnet_info *vi, struct receive_queue *rq, u32 ring_num) { + unsigned int old_ring_num = virtqueue_get_vring_size(rq->vq); + struct xdp_buff **tmp_xsk_buffs = NULL; int err, qindex; qindex = rq - vi->rq; + if (rq->xsk_pool && ring_num > old_ring_num) { + tmp_xsk_buffs = kvzalloc_objs(*tmp_xsk_buffs, ring_num); + if (!tmp_xsk_buffs) + return -ENOMEM; + } + virtnet_rx_pause(vi, rq); err = virtqueue_resize(rq->vq, ring_num, virtnet_rq_unmap_free_buf, NULL); + + /* virtqueue_resize may have changed the size even if err != 0 */ + if (tmp_xsk_buffs && virtqueue_get_vring_size(rq->vq) > old_ring_num) + swap(rq->xsk_buffs, tmp_xsk_buffs); + if (err) netdev_err(vi->dev, "resize rx fail: rx queue index: %d err: %d\n", qindex, err); virtnet_rx_resume(vi, rq, true); + kvfree(tmp_xsk_buffs); return err; } diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c index fbb6ddbb7f89ad..44ea8499e78779 100644 --- a/drivers/net/vxlan/vxlan_core.c +++ b/drivers/net/vxlan/vxlan_core.c @@ -1881,6 +1881,7 @@ static int arp_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni) if (n) { struct vxlan_rdst *rdst = NULL; + u8 ha[ETH_ALEN] __aligned(2); struct vxlan_fdb *f; struct sk_buff *reply; @@ -1889,8 +1890,10 @@ static int arp_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni) goto out; } + neigh_ha_snapshot(ha, n, n->dev); + rcu_read_lock(); - f = vxlan_find_mac_tx(vxlan, n->ha, vni); + f = vxlan_find_mac_tx(vxlan, ha, vni); if (f) rdst = first_remote_rcu(f); if (rdst && vxlan_addr_any(&rdst->remote_ip)) { @@ -1902,7 +1905,7 @@ static int arp_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni) rcu_read_unlock(); reply = arp_create(ARPOP_REPLY, ETH_P_ARP, sip, dev, tip, sha, - n->ha, sha); + ha, sha); neigh_release(n); @@ -1935,7 +1938,8 @@ out: #if IS_ENABLED(CONFIG_IPV6) static struct sk_buff *vxlan_na_create(struct sk_buff *request, - struct neighbour *n, bool isrouter) + struct neighbour *n, u8 *ha, + bool isrouter) { struct net_device *dev = request->dev; struct sk_buff *reply; @@ -1981,7 +1985,7 @@ static struct sk_buff *vxlan_na_create(struct sk_buff *request, /* Ethernet header */ ether_addr_copy(eth_hdr(reply)->h_dest, daddr); - ether_addr_copy(eth_hdr(reply)->h_source, n->ha); + ether_addr_copy(eth_hdr(reply)->h_source, ha); eth_hdr(reply)->h_proto = htons(ETH_P_IPV6); reply->protocol = htons(ETH_P_IPV6); @@ -2010,7 +2014,7 @@ static struct sk_buff *vxlan_na_create(struct sk_buff *request, na->icmph.icmp6_override = 1; na->icmph.icmp6_solicited = 1; na->target = ns->target; - ether_addr_copy(&na->opt[2], n->ha); + ether_addr_copy(&na->opt[2], ha); na->opt[0] = ND_OPT_TARGET_LL_ADDR; na->opt[1] = na_olen >> 3; @@ -2051,6 +2055,7 @@ static int neigh_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni) if (n) { struct vxlan_rdst *rdst = NULL; + u8 ha[ETH_ALEN] __aligned(2); struct vxlan_fdb *f; struct sk_buff *reply; @@ -2059,7 +2064,8 @@ static int neigh_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni) goto out; } - f = vxlan_find_mac_tx(vxlan, n->ha, vni); + neigh_ha_snapshot(ha, n, n->dev); + f = vxlan_find_mac_tx(vxlan, ha, vni); if (f) rdst = first_remote_rcu(f); if (rdst && vxlan_addr_any(&rdst->remote_ip)) { @@ -2068,7 +2074,7 @@ static int neigh_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni) goto out; } - reply = vxlan_na_create(skb, n, + reply = vxlan_na_create(skb, n, ha, !!(f ? f->flags & NTF_ROUTER : 0)); neigh_release(n); diff --git a/drivers/net/vxlan/vxlan_mdb.c b/drivers/net/vxlan/vxlan_mdb.c index 9a9038ae90c18c..d71e1925ecfdbd 100644 --- a/drivers/net/vxlan/vxlan_mdb.c +++ b/drivers/net/vxlan/vxlan_mdb.c @@ -1428,14 +1428,17 @@ static void vxlan_mdb_flush(struct vxlan_dev *vxlan, struct vxlan_mdb_entry *mdb_entry; struct hlist_node *tmp; - /* The removal of an entry cannot trigger the removal of another entry - * since entries are always added to the head of the list. - */ hlist_for_each_entry_safe(mdb_entry, tmp, &vxlan->mdb_list, mdb_node) { if (desc->src_vni && desc->src_vni != mdb_entry->key.vni) continue; vxlan_mdb_remotes_flush(vxlan, mdb_entry, desc); + /* The flush can remove the (S, G) entries created for the + * source list of this entry, including the one saved by + * hlist_for_each_entry_safe(), so re-read it while this entry + * is still linked. + */ + tmp = mdb_entry->mdb_node.next; /* Entry will only be removed if its remotes list is empty. */ vxlan_mdb_entry_put(vxlan, mdb_entry); } diff --git a/drivers/net/vxlan/vxlan_vnifilter.c b/drivers/net/vxlan/vxlan_vnifilter.c index 3e76f4e210944f..dd94085e088656 100644 --- a/drivers/net/vxlan/vxlan_vnifilter.c +++ b/drivers/net/vxlan/vxlan_vnifilter.c @@ -462,10 +462,8 @@ out_err: static const struct nla_policy vni_filter_entry_policy[VXLAN_VNIFILTER_ENTRY_MAX + 1] = { [VXLAN_VNIFILTER_ENTRY_START] = { .type = NLA_U32 }, [VXLAN_VNIFILTER_ENTRY_END] = { .type = NLA_U32 }, - [VXLAN_VNIFILTER_ENTRY_GROUP] = { .type = NLA_BINARY, - .len = sizeof_field(struct iphdr, daddr) }, - [VXLAN_VNIFILTER_ENTRY_GROUP6] = { .type = NLA_BINARY, - .len = sizeof(struct in6_addr) }, + [VXLAN_VNIFILTER_ENTRY_GROUP] = NLA_POLICY_EXACT_LEN(sizeof_field(struct iphdr, daddr)), + [VXLAN_VNIFILTER_ENTRY_GROUP6] = NLA_POLICY_EXACT_LEN(sizeof(struct in6_addr)), }; static const struct nla_policy vni_filter_policy[VXLAN_VNIFILTER_MAX + 1] = { diff --git a/drivers/net/wireless/ath/ath10k/snoc.c b/drivers/net/wireless/ath/ath10k/snoc.c index 3106502275781d..33c98927e8febf 100644 --- a/drivers/net/wireless/ath/ath10k/snoc.c +++ b/drivers/net/wireless/ath/ath10k/snoc.c @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -1475,11 +1476,15 @@ static void ath10k_msa_dump_memory(struct ath10k *ar, hdr->length = cpu_to_le32(ar->msa.mem_size); if (current_region->len < ar->msa.mem_size) { - memcpy(buf, ar->msa.vaddr, current_region->len); + memcpy_fromio(buf, + (const void __iomem __force *)ar->msa.vaddr, + current_region->len); ath10k_warn(ar, "msa dump length is less than msa size %x, %x\n", current_region->len, ar->msa.mem_size); } else { - memcpy(buf, ar->msa.vaddr, ar->msa.mem_size); + memcpy_fromio(buf, + (const void __iomem __force *)ar->msa.vaddr, + ar->msa.mem_size); } } diff --git a/drivers/net/wireless/ath/ath11k/dp_rx.c b/drivers/net/wireless/ath/ath11k/dp_rx.c index 8e2abc7b8383a0..33425707c08426 100644 --- a/drivers/net/wireless/ath/ath11k/dp_rx.c +++ b/drivers/net/wireless/ath/ath11k/dp_rx.c @@ -2334,10 +2334,10 @@ static void ath11k_dp_rx_h_rate(struct ath11k *ar, struct hal_rx_desc *rx_desc, case RX_MSDU_START_PKT_TYPE_11N: rx_status->encoding = RX_ENC_HT; if (rate_mcs > ATH11K_HT_MCS_MAX) { - ath11k_warn(ar->ab, - "Received with invalid mcs in HT mode %d\n", - rate_mcs); - break; + ath11k_dbg(ar->ab, ATH11K_DBG_DP_RX, + "Received HT frame with out-of-range mcs %d, capping to %d\n", + rate_mcs, ATH11K_HT_MCS_MAX); + rate_mcs = ATH11K_HT_MCS_MAX; } rx_status->rate_idx = rate_mcs + (8 * (nss - 1)); if (sgi) @@ -2346,13 +2346,13 @@ static void ath11k_dp_rx_h_rate(struct ath11k *ar, struct hal_rx_desc *rx_desc, break; case RX_MSDU_START_PKT_TYPE_11AC: rx_status->encoding = RX_ENC_VHT; - rx_status->rate_idx = rate_mcs; if (rate_mcs > ATH11K_VHT_MCS_MAX) { - ath11k_warn(ar->ab, - "Received with invalid mcs in VHT mode %d\n", - rate_mcs); - break; + ath11k_dbg(ar->ab, ATH11K_DBG_DP_RX, + "Received VHT frame with out-of-range mcs %d, capping to %d\n", + rate_mcs, ATH11K_VHT_MCS_MAX); + rate_mcs = ATH11K_VHT_MCS_MAX; } + rx_status->rate_idx = rate_mcs; rx_status->nss = nss; if (sgi) rx_status->enc_flags |= RX_ENC_FLAG_SHORT_GI; @@ -2362,14 +2362,14 @@ static void ath11k_dp_rx_h_rate(struct ath11k *ar, struct hal_rx_desc *rx_desc, rx_status->enc_flags |= RX_ENC_FLAG_LDPC; break; case RX_MSDU_START_PKT_TYPE_11AX: - rx_status->rate_idx = rate_mcs; + rx_status->encoding = RX_ENC_HE; if (rate_mcs > ATH11K_HE_MCS_MAX) { - ath11k_warn(ar->ab, - "Received with invalid mcs in HE mode %d\n", - rate_mcs); - break; + ath11k_dbg(ar->ab, ATH11K_DBG_DP_RX, + "Received HE frame with out-of-range mcs %d, capping to %d\n", + rate_mcs, ATH11K_HE_MCS_MAX); + rate_mcs = ATH11K_HE_MCS_MAX; } - rx_status->encoding = RX_ENC_HE; + rx_status->rate_idx = rate_mcs; rx_status->nss = nss; rx_status->he_gi = ath11k_mac_he_gi_to_nl80211_he_gi(sgi); rx_status->bw = ath11k_mac_bw_to_mac80211_bw(bw); diff --git a/drivers/net/wireless/ath/ath11k/wmi.c b/drivers/net/wireless/ath/ath11k/wmi.c index dca6e011cc40e0..e46f4d7ddc80ab 100644 --- a/drivers/net/wireless/ath/ath11k/wmi.c +++ b/drivers/net/wireless/ath/ath11k/wmi.c @@ -159,6 +159,8 @@ static const struct wmi_tlv_policy wmi_tlv_policies[] = { .min_len = sizeof(struct ath11k_wmi_p2p_noa_info) }, [WMI_TAG_P2P_NOA_EVENT] = { .min_len = sizeof(struct wmi_p2p_noa_event) }, + [WMI_TAG_PDEV_CSA_SWITCH_COUNT_STATUS_EVENT] = { + .min_len = sizeof(struct wmi_pdev_csa_switch_ev) }, }; #define PRIMAP(_hw_mode_) \ @@ -262,6 +264,13 @@ const void **ath11k_wmi_tlv_parse_alloc(struct ath11k_base *ab, return tb; } +static u32 ath11k_wmi_tlv_data_len(const void *data) +{ + const struct wmi_tlv *tlv = (const struct wmi_tlv *)data - 1; + + return FIELD_GET(WMI_TLV_LEN, tlv->header); +} + static int ath11k_wmi_cmd_send_nowait(struct ath11k_pdev_wmi *wmi, struct sk_buff *skb, u32 cmd_id) { @@ -2423,8 +2432,8 @@ int ath11k_wmi_send_scan_start_cmd(struct ath11k *ar, for (i = 0; i < params->num_hint_bssid; ++i) { hint_bssid->freq_flags = params->hint_bssid[i].freq_flags; - ether_addr_copy(¶ms->hint_bssid[i].bssid.addr[0], - &hint_bssid->bssid.addr[0]); + ether_addr_copy(&hint_bssid->bssid.addr[0], + ¶ms->hint_bssid[i].bssid.addr[0]); hint_bssid++; } } @@ -4800,14 +4809,16 @@ static int ath11k_wmi_tlv_mac_phy_caps_parse(struct ath11k_base *soc, if (svc_rdy_ext->n_mac_phy_caps >= svc_rdy_ext->tot_phy_id) return -ENOBUFS; - len = min_t(u16, len, sizeof(struct wmi_mac_phy_capabilities)); if (!svc_rdy_ext->n_mac_phy_caps) { - svc_rdy_ext->mac_phy_caps = kcalloc(svc_rdy_ext->tot_phy_id, - len, GFP_ATOMIC); + svc_rdy_ext->mac_phy_caps = + kzalloc_objs(*svc_rdy_ext->mac_phy_caps, + svc_rdy_ext->tot_phy_id, + GFP_ATOMIC); if (!svc_rdy_ext->mac_phy_caps) return -ENOMEM; } + len = min_t(u16, len, sizeof(struct wmi_mac_phy_capabilities)); memcpy(svc_rdy_ext->mac_phy_caps + svc_rdy_ext->n_mac_phy_caps, ptr, len); svc_rdy_ext->n_mac_phy_caps++; return 0; @@ -5118,6 +5129,7 @@ static int ath11k_service_ready_ext_event(struct ath11k_base *ab, return 0; err: + kfree(svc_rdy_ext.mac_phy_caps); ath11k_wmi_free_dbring_caps(ab); return ret; } @@ -8353,15 +8365,23 @@ ath11k_wmi_process_csa_switch_count_event(struct ath11k_base *ab, const struct wmi_pdev_csa_switch_ev *ev, const u32 *vdev_ids) { - int i; + u32 vdev_ids_len = ath11k_wmi_tlv_data_len(vdev_ids); + u32 num_vdevs = ev->num_vdevs; struct ath11k_vif *arvif; + int i; /* Finish CSA once the switch count becomes NULL */ if (ev->current_switch_count) return; + if (num_vdevs > vdev_ids_len / sizeof(*vdev_ids)) { + ath11k_warn(ab, "csa switch count num_vdevs %u exceeds tlv array length %u\n", + num_vdevs, vdev_ids_len); + return; + } + rcu_read_lock(); - for (i = 0; i < ev->num_vdevs; i++) { + for (i = 0; i < num_vdevs; i++) { arvif = ath11k_mac_get_arvif_by_vdev_id(ab, vdev_ids[i]); if (!arvif) { @@ -8895,13 +8915,15 @@ static void ath11k_wmi_tlv_op_rx(struct ath11k_base *ab, struct sk_buff *skb) struct wmi_cmd_hdr *cmd_hdr; enum wmi_tlv_event_id id; + if (skb->len < sizeof(*cmd_hdr)) + goto out; + cmd_hdr = (struct wmi_cmd_hdr *)skb->data; id = FIELD_GET(WMI_CMD_HDR_CMD_ID, (cmd_hdr->cmd_id)); trace_ath11k_wmi_event(ab, id, skb->data, skb->len); - if (skb_pull(skb, sizeof(struct wmi_cmd_hdr)) == NULL) - goto out; + skb_pull(skb, sizeof(*cmd_hdr)); switch (id) { /* Process all the WMI events here */ diff --git a/drivers/net/wireless/ath/ath12k/ahb.c b/drivers/net/wireless/ath/ath12k/ahb.c index 30733a244454e5..6df3b775a21488 100644 --- a/drivers/net/wireless/ath/ath12k/ahb.c +++ b/drivers/net/wireless/ath/ath12k/ahb.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "ahb.h" #include "debug.h" #include "hif.h" @@ -338,24 +339,25 @@ static int ath12k_ahb_power_up(struct ath12k_base *ab) char fw2_name[ATH12K_USERPD_FW_NAME_LEN]; struct device *dev = ab->dev; const struct firmware *fw, *fw2; - struct reserved_mem *rmem = NULL; unsigned long time_left; phys_addr_t mem_phys; + struct resource res; void *mem_region; size_t mem_size; u32 pasid; int ret; - rmem = ath12k_core_get_reserved_mem(ab, 0); - if (!rmem) - return -ENODEV; + ret = of_reserved_mem_region_to_resource_byname(dev->of_node, "q6-region", + &res); + if (ret) + return ret; - mem_phys = rmem->base; - mem_size = rmem->size; + mem_phys = res.start; + mem_size = resource_size(&res); mem_region = devm_memremap(dev, mem_phys, mem_size, MEMREMAP_WC); if (IS_ERR(mem_region)) { - ath12k_err(ab, "unable to map memory region: %pa+%pa\n", - &rmem->base, &rmem->size); + ath12k_err(ab, "unable to map memory region: %pa+%zx\n", + &res.start, mem_size); return PTR_ERR(mem_region); } diff --git a/drivers/net/wireless/ath/ath12k/core.c b/drivers/net/wireless/ath/ath12k/core.c index e87165e4f4b339..5cdf4973d986af 100644 --- a/drivers/net/wireless/ath/ath12k/core.c +++ b/drivers/net/wireless/ath/ath12k/core.c @@ -49,7 +49,7 @@ ath12k_mem_profile_based_param ath12k_mem_profile_based_param[] = { .dp_params = { .tx_comp_ring_size = 32768, .rxdma_monitor_buf_ring_size = 4096, - .rxdma_monitor_dst_ring_size = 8092, + .rxdma_monitor_dst_ring_size = 8192, .num_pool_tx_desc = 32768, .rx_desc_count = 12288, }, @@ -637,31 +637,6 @@ u32 ath12k_core_get_max_peers_per_radio(struct ath12k_base *ab) } EXPORT_SYMBOL(ath12k_core_get_max_peers_per_radio); -struct reserved_mem *ath12k_core_get_reserved_mem(struct ath12k_base *ab, - int index) -{ - struct device *dev = ab->dev; - struct reserved_mem *rmem; - struct device_node *node; - - node = of_parse_phandle(dev->of_node, "memory-region", index); - if (!node) { - ath12k_dbg(ab, ATH12K_DBG_BOOT, - "failed to parse memory-region for index %d\n", index); - return NULL; - } - - rmem = of_reserved_mem_lookup(node); - of_node_put(node); - if (!rmem) { - ath12k_dbg(ab, ATH12K_DBG_BOOT, - "unable to get memory-region for index %d\n", index); - return NULL; - } - - return rmem; -} - static inline void ath12k_core_to_group_ref_get(struct ath12k_base *ab) { @@ -1524,7 +1499,7 @@ static void ath12k_core_pre_reconfigure_recovery(struct ath12k_base *ab) complete_all(&ar->scan.completed); complete(&ar->scan.on_channel); complete(&ar->peer_assoc_done); - complete(&ar->peer_delete_done); + ath12k_peer_delete_wait_flush(ar); complete(&ar->install_key_done); complete(&ar->vdev_setup_done); complete(&ar->vdev_delete_done); diff --git a/drivers/net/wireless/ath/ath12k/core.h b/drivers/net/wireless/ath/ath12k/core.h index 30726e580833e1..f28d2e90b67b29 100644 --- a/drivers/net/wireless/ath/ath12k/core.h +++ b/drivers/net/wireless/ath/ath12k/core.h @@ -666,7 +666,8 @@ struct ath12k { /* protects the radio specific data like debug stats, ppdu_stats_info stats, * vdev_stop_status info, scan data, ath12k_sta info, ath12k_link_vif info, - * channel context data, survey info, test mode data, regd_channel_update_queue. + * channel context data, test mode data, regd_channel_update_queue, + * peer_delete_waits. */ spinlock_t data_lock; @@ -688,7 +689,7 @@ struct ath12k { u8 radio_idx; struct completion peer_assoc_done; - struct completion peer_delete_done; + struct list_head peer_delete_waits; int install_key_status; struct completion install_key_done; @@ -722,7 +723,6 @@ struct ath12k { * avoid reporting garbage data. */ bool ch_info_can_report_survey; - struct survey_info survey[ATH12K_NUM_CHANS]; struct completion bss_survey_done; struct work_struct regd_update_work; @@ -792,6 +792,11 @@ struct ath12k_hw { */ struct mutex hw_mutex; enum ath12k_hw_state state; + + /* protects survey[] shared across radios of this hw. */ + spinlock_t survey_lock; + struct survey_info survey[ATH12K_NUM_CHANS]; + bool regd_updated; bool use_6ghz_regd; bool host_alloc_ml_id; @@ -1297,8 +1302,6 @@ void ath12k_fw_stats_init(struct ath12k *ar); void ath12k_fw_stats_bcn_free(struct list_head *head); void ath12k_fw_stats_free(struct ath12k_fw_stats *stats); void ath12k_fw_stats_reset(struct ath12k *ar); -struct reserved_mem *ath12k_core_get_reserved_mem(struct ath12k_base *ab, - int index); enum ath12k_qmi_mem_mode ath12k_core_get_memory_mode(struct ath12k_base *ab); static inline const char *ath12k_scan_state_str(enum ath12k_scan_state state) diff --git a/drivers/net/wireless/ath/ath12k/dp_tx.c b/drivers/net/wireless/ath/ath12k/dp_tx.c index c10da6195c9c34..9644f9ef2c7494 100644 --- a/drivers/net/wireless/ath/ath12k/dp_tx.c +++ b/drivers/net/wireless/ath/ath12k/dp_tx.c @@ -82,6 +82,52 @@ enum hal_encrypt_type ath12k_dp_tx_get_encrypt_type(u32 cipher) } EXPORT_SYMBOL(ath12k_dp_tx_get_encrypt_type); +u8 ath12k_dp_tx_crypto_iv_len(enum hal_encrypt_type enc_type) +{ + switch (enc_type) { + case HAL_ENCRYPT_TYPE_TKIP_NO_MIC: + case HAL_ENCRYPT_TYPE_TKIP_MIC: + return IEEE80211_TKIP_IV_LEN; + case HAL_ENCRYPT_TYPE_CCMP_128: + return IEEE80211_CCMP_HDR_LEN; + case HAL_ENCRYPT_TYPE_CCMP_256: + return IEEE80211_CCMP_256_HDR_LEN; + case HAL_ENCRYPT_TYPE_GCMP_128: + case HAL_ENCRYPT_TYPE_AES_GCMP_256: + return IEEE80211_GCMP_HDR_LEN; + case HAL_ENCRYPT_TYPE_WEP_40: + case HAL_ENCRYPT_TYPE_WEP_104: + case HAL_ENCRYPT_TYPE_WEP_128: + return IEEE80211_WEP_IV_LEN; + default: + return 0; + } +} +EXPORT_SYMBOL(ath12k_dp_tx_crypto_iv_len); + +u8 ath12k_dp_tx_crypto_icv_len(enum hal_encrypt_type enc_type) +{ + switch (enc_type) { + case HAL_ENCRYPT_TYPE_CCMP_128: + return IEEE80211_CCMP_MIC_LEN; + case HAL_ENCRYPT_TYPE_CCMP_256: + return IEEE80211_CCMP_256_MIC_LEN; + case HAL_ENCRYPT_TYPE_GCMP_128: + case HAL_ENCRYPT_TYPE_AES_GCMP_256: + return IEEE80211_GCMP_MIC_LEN; + case HAL_ENCRYPT_TYPE_TKIP_NO_MIC: + case HAL_ENCRYPT_TYPE_TKIP_MIC: + return IEEE80211_TKIP_ICV_LEN; + case HAL_ENCRYPT_TYPE_WEP_40: + case HAL_ENCRYPT_TYPE_WEP_104: + case HAL_ENCRYPT_TYPE_WEP_128: + return IEEE80211_WEP_ICV_LEN; + default: + return 0; + } +} +EXPORT_SYMBOL(ath12k_dp_tx_crypto_icv_len); + void ath12k_dp_tx_release_txbuf(struct ath12k_dp *dp, struct ath12k_tx_desc_info *tx_desc, u8 pool_id) diff --git a/drivers/net/wireless/ath/ath12k/dp_tx.h b/drivers/net/wireless/ath/ath12k/dp_tx.h index 7cef20540179fa..1af79af2ada265 100644 --- a/drivers/net/wireless/ath/ath12k/dp_tx.h +++ b/drivers/net/wireless/ath/ath12k/dp_tx.h @@ -19,6 +19,8 @@ enum hal_tcl_encap_type ath12k_dp_tx_get_encap_type(struct ath12k_base *ab, struct sk_buff *skb); void ath12k_dp_tx_encap_nwifi(struct sk_buff *skb); u8 ath12k_dp_tx_get_tid(struct sk_buff *skb); +u8 ath12k_dp_tx_crypto_iv_len(enum hal_encrypt_type enc_type); +u8 ath12k_dp_tx_crypto_icv_len(enum hal_encrypt_type enc_type); void *ath12k_dp_metadata_align_skb(struct sk_buff *skb, u8 tail_len); int ath12k_dp_tx_align_payload(struct ath12k_dp *dp, struct sk_buff **pskb); void ath12k_dp_tx_release_txbuf(struct ath12k_dp *dp, diff --git a/drivers/net/wireless/ath/ath12k/hal.c b/drivers/net/wireless/ath/ath12k/hal.c index a164563fff289c..f03817b2fbc52d 100644 --- a/drivers/net/wireless/ath/ath12k/hal.c +++ b/drivers/net/wireless/ath/ath12k/hal.c @@ -828,8 +828,8 @@ void *ath12k_hal_encode_tlv64_hdr(void *tlv, u64 tag, u64 len) { struct hal_tlv_64_hdr *tlv64 = tlv; - tlv64->tl = le64_encode_bits(tag, HAL_TLV_HDR_TAG) | - le64_encode_bits(len, HAL_TLV_HDR_LEN); + tlv64->tl = le64_encode_bits(tag, HAL_TLV_64_HDR_TAG) | + le64_encode_bits(len, HAL_TLV_64_HDR_LEN); return tlv64->value; } @@ -851,7 +851,7 @@ u16 ath12k_hal_decode_tlv64_hdr(void *tlv, void **desc) struct hal_tlv_64_hdr *tlv64 = tlv; u16 tag; - tag = le64_get_bits(tlv64->tl, HAL_SRNG_TLV_HDR_TAG); + tag = le64_get_bits(tlv64->tl, HAL_TLV_64_HDR_TAG); *desc = tlv64->value; return tag; @@ -863,7 +863,7 @@ u16 ath12k_hal_decode_tlv32_hdr(void *tlv, void **desc) struct hal_tlv_hdr *tlv32 = tlv; u16 tag; - tag = le32_get_bits(tlv32->tl, HAL_SRNG_TLV_HDR_TAG); + tag = le32_get_bits(tlv32->tl, HAL_TLV_HDR_TAG); *desc = tlv32->value; return tag; diff --git a/drivers/net/wireless/ath/ath12k/hal.h b/drivers/net/wireless/ath/ath12k/hal.h index 21c551d8b24814..3ee49d93e24a09 100644 --- a/drivers/net/wireless/ath/ath12k/hal.h +++ b/drivers/net/wireless/ath/ath12k/hal.h @@ -1444,7 +1444,7 @@ struct hal_ops { }; #define HAL_TLV_HDR_TAG GENMASK(9, 1) -#define HAL_TLV_HDR_LEN GENMASK(25, 10) +#define HAL_TLV_HDR_LEN GENMASK(21, 10) #define HAL_TLV_USR_ID GENMASK(31, 26) #define HAL_TLV_ALIGN 4 @@ -1464,9 +1464,6 @@ struct hal_tlv_64_hdr { u8 value[]; } __packed; -#define HAL_SRNG_TLV_HDR_TAG GENMASK(9, 1) -#define HAL_SRNG_TLV_HDR_LEN GENMASK(25, 10) - dma_addr_t ath12k_hal_srng_get_tp_addr(struct ath12k_base *ab, struct hal_srng *srng); dma_addr_t ath12k_hal_srng_get_hp_addr(struct ath12k_base *ab, diff --git a/drivers/net/wireless/ath/ath12k/mac.c b/drivers/net/wireless/ath/ath12k/mac.c index a0928890671ac6..b24f97786c5907 100644 --- a/drivers/net/wireless/ath/ath12k/mac.c +++ b/drivers/net/wireless/ath/ath12k/mac.c @@ -10169,16 +10169,16 @@ static void ath12k_mac_update_vif_offload(struct ath12k_link_vif *arvif) if (vif->type != NL80211_IFTYPE_STATION && vif->type != NL80211_IFTYPE_AP) vif->offload_flags &= ~(IEEE80211_OFFLOAD_ENCAP_ENABLED | - IEEE80211_OFFLOAD_DECAP_ENABLED); + IEEE80211_OFFLOAD_DECAP_ENABLED | + IEEE80211_OFFLOAD_ENCAP_MCAST | + IEEE80211_OFFLOAD_ENCAP_4ADDR); - if (vif->offload_flags & IEEE80211_OFFLOAD_ENCAP_ENABLED) { + if (vif->offload_flags & IEEE80211_OFFLOAD_ENCAP_ENABLED) ahvif->dp_vif.tx_encap_type = ATH12K_HW_TXRX_ETHERNET; - vif->offload_flags |= IEEE80211_OFFLOAD_ENCAP_4ADDR; - } else if (test_bit(ATH12K_FLAG_RAW_MODE, &ab->dev_flags)) { + else if (test_bit(ATH12K_FLAG_RAW_MODE, &ab->dev_flags)) ahvif->dp_vif.tx_encap_type = ATH12K_HW_TXRX_RAW; - } else { + else ahvif->dp_vif.tx_encap_type = ATH12K_HW_TXRX_NATIVE_WIFI; - } ret = ath12k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id, param_id, ahvif->dp_vif.tx_encap_type); @@ -10188,6 +10188,10 @@ static void ath12k_mac_update_vif_offload(struct ath12k_link_vif *arvif) vif->offload_flags &= ~IEEE80211_OFFLOAD_ENCAP_ENABLED; } + if (vif->offload_flags & IEEE80211_OFFLOAD_ENCAP_ENABLED) + vif->offload_flags |= (IEEE80211_OFFLOAD_ENCAP_MCAST | + IEEE80211_OFFLOAD_ENCAP_4ADDR); + param_id = WMI_VDEV_PARAM_RX_DECAP_TYPE; if (vif->offload_flags & IEEE80211_OFFLOAD_DECAP_ENABLED) param_value = ATH12K_HW_TXRX_ETHERNET; @@ -10616,22 +10620,8 @@ int ath12k_mac_vdev_create(struct ath12k *ar, struct ath12k_link_vif *arvif) err_peer_del: if (ahvif->vdev_type == WMI_VDEV_TYPE_AP) { - reinit_completion(&ar->peer_delete_done); - - ret = ath12k_wmi_send_peer_delete_cmd(ar, arvif->bssid, - arvif->vdev_id); - if (ret) { - ath12k_warn(ar->ab, "failed to delete peer vdev_id %d addr %pM\n", - arvif->vdev_id, arvif->bssid); - goto err_dp_peer_del; - } - - ret = ath12k_wait_for_peer_delete_done(ar, arvif->vdev_id, - arvif->bssid); - if (ret) - goto err_dp_peer_del; - - ar->num_peers--; + /* ignore return value: propagate the original error */ + ath12k_peer_delete(ar, arvif->vdev_id, arvif->bssid); } err_dp_peer_del: @@ -11305,6 +11295,8 @@ ath12k_mac_mlo_get_vdev_args(struct ath12k_link_vif *arvif, ml_arg->assoc_link = arvif->is_sta_assoc_link; + ml_arg->ieee_link_id = arvif->link_id; + partner_info = ml_arg->partner_info; links = ahvif->links_map; @@ -11328,6 +11320,7 @@ ath12k_mac_mlo_get_vdev_args(struct ath12k_link_vif *arvif, partner_info->vdev_id = arvif_p->vdev_id; partner_info->hw_link_id = arvif_p->ar->pdev->hw_link_id; + partner_info->ieee_link_id = arvif_p->link_id; ether_addr_copy(partner_info->addr, link_conf->addr); ml_arg->num_partner_links++; partner_info++; @@ -13633,52 +13626,54 @@ ath12k_mac_update_bss_chan_survey(struct ath12k *ar, int ath12k_mac_op_get_survey(struct ieee80211_hw *hw, int idx, struct survey_info *survey) { + struct ath12k_hw *ah = hw->priv; struct ath12k *ar; struct ieee80211_supported_band *sband; - struct survey_info *ar_survey; + struct survey_info *ah_survey; + int sband_idx = idx; lockdep_assert_wiphy(hw->wiphy); - if (idx >= ATH12K_NUM_CHANS) + if (sband_idx >= ATH12K_NUM_CHANS) return -ENOENT; sband = hw->wiphy->bands[NL80211_BAND_2GHZ]; - if (sband && idx >= sband->n_channels) { - idx -= sband->n_channels; + if (sband && sband_idx >= sband->n_channels) { + sband_idx -= sband->n_channels; sband = NULL; } if (!sband) sband = hw->wiphy->bands[NL80211_BAND_5GHZ]; - if (sband && idx >= sband->n_channels) { - idx -= sband->n_channels; + if (sband && sband_idx >= sband->n_channels) { + sband_idx -= sband->n_channels; sband = NULL; } if (!sband) sband = hw->wiphy->bands[NL80211_BAND_6GHZ]; - if (!sband || idx >= sband->n_channels) + if (!sband || sband_idx >= sband->n_channels) return -ENOENT; - ar = ath12k_mac_get_ar_by_chan(hw, &sband->channels[idx]); + ar = ath12k_mac_get_ar_by_chan(hw, &sband->channels[sband_idx]); if (!ar) { - if (sband->channels[idx].flags & IEEE80211_CHAN_DISABLED) { + if (sband->channels[sband_idx].flags & IEEE80211_CHAN_DISABLED) { memset(survey, 0, sizeof(*survey)); return 0; } return -ENOENT; } - ar_survey = &ar->survey[idx]; + ah_survey = &ah->survey[idx]; - ath12k_mac_update_bss_chan_survey(ar, &sband->channels[idx]); + ath12k_mac_update_bss_chan_survey(ar, &sband->channels[sband_idx]); - spin_lock_bh(&ar->data_lock); - memcpy(survey, ar_survey, sizeof(*survey)); - spin_unlock_bh(&ar->data_lock); + scoped_guard(spinlock_bh, &ah->survey_lock) { + memcpy(survey, ah_survey, sizeof(*survey)); + } - survey->channel = &sband->channels[idx]; + survey->channel = &sband->channels[sband_idx]; if (ar->rx_channel == survey->channel) survey->filled |= SURVEY_INFO_IN_USE; @@ -15106,11 +15101,11 @@ static void ath12k_mac_setup(struct ath12k *ar) spin_lock_init(&ar->dp.ppdu_list_lock); INIT_LIST_HEAD(&ar->arvifs); INIT_LIST_HEAD(&ar->dp.ppdu_stats_info); + INIT_LIST_HEAD(&ar->peer_delete_waits); init_completion(&ar->vdev_setup_done); init_completion(&ar->vdev_delete_done); init_completion(&ar->peer_assoc_done); - init_completion(&ar->peer_delete_done); init_completion(&ar->install_key_done); init_completion(&ar->bss_survey_done); init_completion(&ar->scan.started); @@ -15360,6 +15355,7 @@ static struct ath12k_hw *ath12k_mac_hw_allocate(struct ath12k_hw_group *ag, mutex_init(&ah->hw_mutex); init_completion(&ah->peer_ml_id_done); + spin_lock_init(&ah->survey_lock); spin_lock_init(&ah->dp_hw.peer_lock); INIT_LIST_HEAD(&ah->dp_hw.dp_peers_list); diff --git a/drivers/net/wireless/ath/ath12k/peer.c b/drivers/net/wireless/ath/ath12k/peer.c index ed0524ddff803c..80edebf0e364b1 100644 --- a/drivers/net/wireless/ath/ath12k/peer.c +++ b/drivers/net/wireless/ath/ath12k/peer.c @@ -9,6 +9,55 @@ #include "debug.h" #include "debugfs.h" +static void ath12k_peer_delete_wait_register(struct ath12k *ar, + struct ath12k_peer_delete_wait *wait, + u32 vdev_id, const u8 *addr) +{ + wait->vdev_id = vdev_id; + ether_addr_copy(wait->addr, addr); + init_completion(&wait->done); + + spin_lock_bh(&ar->data_lock); + list_add(&wait->list, &ar->peer_delete_waits); + spin_unlock_bh(&ar->data_lock); +} + +static void ath12k_peer_delete_wait_unregister(struct ath12k *ar, + struct ath12k_peer_delete_wait *wait) +{ + spin_lock_bh(&ar->data_lock); + list_del(&wait->list); + spin_unlock_bh(&ar->data_lock); +} + +void ath12k_peer_delete_resp_signal(struct ath12k *ar, u32 vdev_id, const u8 *addr) +{ + struct ath12k_peer_delete_wait *wait; + + guard(spinlock_bh)(&ar->data_lock); + + list_for_each_entry(wait, &ar->peer_delete_waits, list) { + if (wait->vdev_id == vdev_id && + ether_addr_equal(wait->addr, addr)) { + complete(&wait->done); + return; + } + } + + ath12k_warn(ar->ab, "failed to find link peer with vdev id %u addr %pM\n", + vdev_id, addr); +} + +void ath12k_peer_delete_wait_flush(struct ath12k *ar) +{ + struct ath12k_peer_delete_wait *wait; + + spin_lock_bh(&ar->data_lock); + list_for_each_entry(wait, &ar->peer_delete_waits, list) + complete(&wait->done); + spin_unlock_bh(&ar->data_lock); +} + static int ath12k_wait_for_dp_link_peer_common(struct ath12k_base *ab, int vdev_id, const u8 *addr, bool expect_mapped) { @@ -62,20 +111,19 @@ static int ath12k_wait_for_peer_deleted(struct ath12k *ar, int vdev_id, const u8 return ath12k_wait_for_dp_link_peer_common(ar->ab, vdev_id, addr, false); } -int ath12k_wait_for_peer_delete_done(struct ath12k *ar, u32 vdev_id, - const u8 *addr) +int ath12k_wait_for_peer_delete_done(struct ath12k *ar, + struct ath12k_peer_delete_wait *wait) { - int ret; unsigned long time_left; + int ret; - ret = ath12k_wait_for_peer_deleted(ar, vdev_id, addr); + ret = ath12k_wait_for_peer_deleted(ar, wait->vdev_id, wait->addr); if (ret) { - ath12k_warn(ar->ab, "failed wait for peer deleted"); + ath12k_warn(ar->ab, "failed wait for peer deleted\n"); return ret; } - time_left = wait_for_completion_timeout(&ar->peer_delete_done, - 3 * HZ); + time_left = wait_for_completion_timeout(&wait->done, 3 * HZ); if (time_left == 0) { ath12k_warn(ar->ab, "Timeout in receiving peer delete response\n"); return -ETIMEDOUT; @@ -91,8 +139,6 @@ static int ath12k_peer_delete_send(struct ath12k *ar, u32 vdev_id, const u8 *add lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); - reinit_completion(&ar->peer_delete_done); - ret = ath12k_wmi_send_peer_delete_cmd(ar, addr, vdev_id); if (ret) { ath12k_warn(ab, @@ -106,6 +152,7 @@ static int ath12k_peer_delete_send(struct ath12k *ar, u32 vdev_id, const u8 *add int ath12k_peer_delete(struct ath12k *ar, u32 vdev_id, u8 *addr) { + struct ath12k_peer_delete_wait wait; int ret; lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); @@ -114,17 +161,25 @@ int ath12k_peer_delete(struct ath12k *ar, u32 vdev_id, u8 *addr) &(ath12k_ar_to_ah(ar)->dp_hw), vdev_id, addr, ar->hw_link_id); + /* + * Register the stack waiter before sending so the resp_event for + * this peer cannot arrive while no waiter is queued. + */ + ath12k_peer_delete_wait_register(ar, &wait, vdev_id, addr); + ret = ath12k_peer_delete_send(ar, vdev_id, addr); if (ret) - return ret; + goto out; - ret = ath12k_wait_for_peer_delete_done(ar, vdev_id, addr); + ret = ath12k_wait_for_peer_delete_done(ar, &wait); if (ret) - return ret; + goto out; ar->num_peers--; - return 0; +out: + ath12k_peer_delete_wait_unregister(ar, &wait); + return ret; } static int ath12k_wait_for_peer_created(struct ath12k *ar, int vdev_id, const u8 *addr) @@ -184,22 +239,26 @@ int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif, peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arg->vdev_id, arg->peer_addr); if (!peer) { + struct ath12k_peer_delete_wait wait; + spin_unlock_bh(&dp->dp_lock); ath12k_warn(ar->ab, "failed to find peer %pM on vdev %i after creation\n", arg->peer_addr, arg->vdev_id); - reinit_completion(&ar->peer_delete_done); + ath12k_peer_delete_wait_register(ar, &wait, arg->vdev_id, + arg->peer_addr); ret = ath12k_wmi_send_peer_delete_cmd(ar, arg->peer_addr, arg->vdev_id); if (ret) { ath12k_warn(ar->ab, "failed to delete peer vdev_id %d addr %pM\n", arg->vdev_id, arg->peer_addr); + ath12k_peer_delete_wait_unregister(ar, &wait); return ret; } - ret = ath12k_wait_for_peer_delete_done(ar, arg->vdev_id, - arg->peer_addr); + ret = ath12k_wait_for_peer_delete_done(ar, &wait); + ath12k_peer_delete_wait_unregister(ar, &wait); if (ret) return ret; @@ -308,13 +367,14 @@ void ath12k_peer_ml_free(struct ath12k_hw *ah, struct ath12k_sta *ahsta) int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_sta *ahsta) { + DECLARE_BITMAP(registered, IEEE80211_MLD_MAX_NUM_LINKS); struct ieee80211_sta *sta = ath12k_ahsta_to_sta(ahsta); struct ath12k_hw *ah = ahvif->ah; struct ath12k_link_vif *arvif; struct ath12k_link_sta *arsta; + int ret, err_ret = 0; unsigned long links; struct ath12k *ar; - int ret, err_ret = 0; u8 link_id; lockdep_assert_wiphy(ah->hw->wiphy); @@ -322,8 +382,19 @@ int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_st if (!sta->mlo) return -EINVAL; - /* FW expects delete of all link peers at once before waiting for reception - * of peer unmap or delete responses + struct ath12k_peer_delete_wait *waits __free(kfree) = + kzalloc_objs(*waits, IEEE80211_MLD_MAX_NUM_LINKS); + if (!waits) + return -ENOMEM; + + bitmap_zero(registered, IEEE80211_MLD_MAX_NUM_LINKS); + + /* + * Firmware expects delete of all link peers at once before waiting + * for reception of peer unmap or delete responses. Phase 1 registers + * a per-link stack waiter and sends WMI peer delete for every + * link; the resp_event handler matches each response to its + * (vdev_id, addr) waiter on ar->peer_delete_waits. */ links = ahsta->links_map; for_each_set_bit(link_id, &links, IEEE80211_MLD_MAX_NUM_LINKS) { @@ -343,29 +414,36 @@ int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_st arvif->vdev_id, arsta->addr, ar->hw_link_id); + ath12k_peer_delete_wait_register(ar, &waits[link_id], + arvif->vdev_id, arsta->addr); + ret = ath12k_peer_delete_send(ar, arvif->vdev_id, arsta->addr); if (ret) { ath12k_warn(ar->ab, "failed to delete peer vdev_id %d addr %pM ret %d\n", arvif->vdev_id, arsta->addr, ret); err_ret = ret; + ath12k_peer_delete_wait_unregister(ar, &waits[link_id]); continue; } + + set_bit(link_id, registered); } - /* Ensure all link peers are deleted and unmapped */ + /* + * Phase 2: wait for unmap + delete_resp on each registered link + * and tear down the waiter. + */ links = ahsta->links_map; for_each_set_bit(link_id, &links, IEEE80211_MLD_MAX_NUM_LINKS) { - arvif = wiphy_dereference(ah->hw->wiphy, ahvif->link[link_id]); - arsta = wiphy_dereference(ah->hw->wiphy, ahsta->link[link_id]); - if (!arvif || !arsta) + if (!test_bit(link_id, registered)) continue; + arvif = wiphy_dereference(ah->hw->wiphy, ahvif->link[link_id]); ar = arvif->ar; - if (!ar) - continue; - ret = ath12k_wait_for_peer_delete_done(ar, arvif->vdev_id, arsta->addr); + ret = ath12k_wait_for_peer_delete_done(ar, &waits[link_id]); + ath12k_peer_delete_wait_unregister(ar, &waits[link_id]); if (ret) { err_ret = ret; continue; diff --git a/drivers/net/wireless/ath/ath12k/peer.h b/drivers/net/wireless/ath/ath12k/peer.h index 0f7f25b8e89c03..3f4ac17b9aa659 100644 --- a/drivers/net/wireless/ath/ath12k/peer.h +++ b/drivers/net/wireless/ath/ath12k/peer.h @@ -9,13 +9,23 @@ #include "dp_peer.h" +struct ath12k_peer_delete_wait { + struct list_head list; + u32 vdev_id; + u8 addr[ETH_ALEN]; + struct completion done; +}; + +void ath12k_peer_delete_resp_signal(struct ath12k *ar, u32 vdev_id, const u8 *addr); +void ath12k_peer_delete_wait_flush(struct ath12k *ar); + void ath12k_peer_cleanup(struct ath12k *ar, u32 vdev_id); int ath12k_peer_delete(struct ath12k *ar, u32 vdev_id, u8 *addr); int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif, struct ieee80211_sta *sta, struct ath12k_wmi_peer_create_arg *arg); -int ath12k_wait_for_peer_delete_done(struct ath12k *ar, u32 vdev_id, - const u8 *addr); +int ath12k_wait_for_peer_delete_done(struct ath12k *ar, + struct ath12k_peer_delete_wait *wait); int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_sta *ahsta); struct ath12k_ml_peer *ath12k_peer_ml_find(struct ath12k_hw *ah, const u8 *addr); diff --git a/drivers/net/wireless/ath/ath12k/qmi.c b/drivers/net/wireless/ath/ath12k/qmi.c index fd762b5d7bb597..6d506322922143 100644 --- a/drivers/net/wireless/ath/ath12k/qmi.c +++ b/drivers/net/wireless/ath/ath12k/qmi.c @@ -13,6 +13,7 @@ #include #include #include +#include #define SLEEP_CLOCK_SELECT_INTERNAL_BIT 0x02 #define HOST_CSTATE_BIT 0x04 @@ -2725,121 +2726,96 @@ err: return ret; } +static const char *ath12k_qmi_get_mem_reg_name(int mem_type) +{ + switch (mem_type) { + case HOST_DDR_REGION_TYPE: + case BDF_MEM_REGION_TYPE: + return "q6-region"; + case M3_DUMP_REGION_TYPE: + return "m3-dump"; + case CALDB_MEM_REGION_TYPE: + return "q6-caldb"; + case MLO_GLOBAL_MEM_REGION_TYPE: + return "mlo-global-mem"; + default: + return NULL; + } +} + static int ath12k_qmi_assign_target_mem_chunk(struct ath12k_base *ab) { - struct reserved_mem *rmem; - size_t avail_rmem_size; + struct device_node *np = ab->dev->of_node; + size_t avail_rmem_size, offset = 0; + struct target_mem_chunk *chunk; + struct resource res; + const char *rname; int i, idx, ret; for (i = 0, idx = 0; i < ab->qmi.mem_seg_count; i++) { - switch (ab->qmi.target_mem[i].type) { - case HOST_DDR_REGION_TYPE: - rmem = ath12k_core_get_reserved_mem(ab, 0); - if (!rmem) { - ret = -ENODEV; - goto out; - } - - avail_rmem_size = rmem->size; - if (avail_rmem_size < ab->qmi.target_mem[i].size) { - ath12k_dbg(ab, ATH12K_DBG_QMI, - "failed to assign mem type %u req size %u avail size %zu\n", - ab->qmi.target_mem[i].type, - ab->qmi.target_mem[i].size, - avail_rmem_size); - ret = -EINVAL; - goto out; - } - - ab->qmi.target_mem[idx].paddr = rmem->base; - ab->qmi.target_mem[idx].v.ioaddr = - ioremap(ab->qmi.target_mem[idx].paddr, - ab->qmi.target_mem[i].size); - if (!ab->qmi.target_mem[idx].v.ioaddr) { - ret = -EIO; - goto out; - } - ab->qmi.target_mem[idx].size = ab->qmi.target_mem[i].size; - ab->qmi.target_mem[idx].type = ab->qmi.target_mem[i].type; - idx++; - break; - case BDF_MEM_REGION_TYPE: - rmem = ath12k_core_get_reserved_mem(ab, 0); - if (!rmem) { - ret = -ENODEV; - goto out; - } - - avail_rmem_size = rmem->size - ab->hw_params->bdf_addr_offset; - if (avail_rmem_size < ab->qmi.target_mem[i].size) { - ath12k_dbg(ab, ATH12K_DBG_QMI, - "failed to assign mem type %u req size %u avail size %zu\n", - ab->qmi.target_mem[i].type, - ab->qmi.target_mem[i].size, - avail_rmem_size); - ret = -EINVAL; - goto out; - } - ab->qmi.target_mem[idx].paddr = - rmem->base + ab->hw_params->bdf_addr_offset; - ab->qmi.target_mem[idx].v.ioaddr = - ioremap(ab->qmi.target_mem[idx].paddr, - ab->qmi.target_mem[i].size); - if (!ab->qmi.target_mem[idx].v.ioaddr) { - ret = -EIO; - goto out; - } - ab->qmi.target_mem[idx].size = ab->qmi.target_mem[i].size; - ab->qmi.target_mem[idx].type = ab->qmi.target_mem[i].type; - idx++; - break; - case CALDB_MEM_REGION_TYPE: - /* Cold boot calibration is not enabled in Ath12k. Hence, + chunk = &ab->qmi.target_mem[i]; + if (chunk->type == CALDB_MEM_REGION_TYPE) { + /* + * Cold boot calibration is not enabled in Ath12k. Hence, * assign paddr = 0. * Once cold boot calibration is enabled add support to * assign reserved memory from DT. */ ab->qmi.target_mem[idx].paddr = 0; ab->qmi.target_mem[idx].v.ioaddr = NULL; - ab->qmi.target_mem[idx].size = ab->qmi.target_mem[i].size; - ab->qmi.target_mem[idx].type = ab->qmi.target_mem[i].type; + ab->qmi.target_mem[idx].size = chunk->size; + ab->qmi.target_mem[idx].type = chunk->type; idx++; - break; - case M3_DUMP_REGION_TYPE: - rmem = ath12k_core_get_reserved_mem(ab, 1); - if (!rmem) { - ret = -EINVAL; - goto out; - } + continue; + } - avail_rmem_size = rmem->size; - if (avail_rmem_size < ab->qmi.target_mem[i].size) { - ath12k_dbg(ab, ATH12K_DBG_QMI, - "failed to assign mem type %u req size %u avail size %zu\n", - ab->qmi.target_mem[i].type, - ab->qmi.target_mem[i].size, + rname = ath12k_qmi_get_mem_reg_name(chunk->type); + if (!rname) { + ath12k_warn(ab, "qmi ignore invalid mem req type %u\n", + chunk->type); + continue; + } + + ret = of_reserved_mem_region_to_resource_byname(np, rname, &res); + if (ret) + goto out; + + avail_rmem_size = resource_size(&res); + if (chunk->type == BDF_MEM_REGION_TYPE || + chunk->type == HOST_DDR_REGION_TYPE) { + if (ab->hw_params->bdf_addr_offset > avail_rmem_size || + offset > avail_rmem_size - ab->hw_params->bdf_addr_offset) { + ath12k_err(ab, "qmi mem offset overflow: bdf_offset=%u offset=%zu size=%zu\n", + ab->hw_params->bdf_addr_offset, offset, avail_rmem_size); ret = -EINVAL; goto out; } - ab->qmi.target_mem[idx].paddr = rmem->base; - ab->qmi.target_mem[idx].v.ioaddr = - ioremap(ab->qmi.target_mem[idx].paddr, - ab->qmi.target_mem[i].size); - if (!ab->qmi.target_mem[idx].v.ioaddr) { - ret = -EIO; - goto out; - } - ab->qmi.target_mem[idx].size = ab->qmi.target_mem[i].size; - ab->qmi.target_mem[idx].type = ab->qmi.target_mem[i].type; - idx++; - break; - default: - ath12k_warn(ab, "qmi ignore invalid mem req type %u\n", - ab->qmi.target_mem[i].type); - break; + avail_rmem_size -= ab->hw_params->bdf_addr_offset + offset; + res.start += ab->hw_params->bdf_addr_offset + offset; + offset += chunk->size; } + + if (avail_rmem_size < chunk->size) { + ath12k_dbg(ab, ATH12K_DBG_QMI, + "failed to assign mem type %u req size %u avail size %zu\n", + chunk->type, chunk->size, avail_rmem_size); + ret = -EINVAL; + goto out; + } + + ab->qmi.target_mem[idx].paddr = res.start; + ab->qmi.target_mem[idx].v.ioaddr = ioremap(ab->qmi.target_mem[idx].paddr, + chunk->size); + if (!ab->qmi.target_mem[idx].v.ioaddr) { + ret = -EIO; + goto out; + } + + ab->qmi.target_mem[idx].size = chunk->size; + ab->qmi.target_mem[idx].type = chunk->type; + idx++; } ab->qmi.mem_seg_count = idx; diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c b/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c index d2749de445534a..587d58eeccfa5d 100644 --- a/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c +++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c @@ -13,6 +13,49 @@ #include "hal.h" #include "hal_tx.h" +/* + * Convert an encrypted EAPOL frame from native-WiFi format to + * the layout expected by the firmware RAW encrypt pipeline: + * + * [802.11 hdr][IV (zeroed)][LLC/SNAP][EAPOL payload][ICV (zeroed)] + * + * mac80211 delivers the frame as [802.11 hdr][LLC/SNAP][EAPOL payload]. + * The MAC header length is read from the unmodified skb and is safe because + * ieee80211_hdrlen() only inspects the 2-byte frame_control field. + * pskb_expand_head() is used to grow both head (for the IV) and tail + * (for the ICV) in a single call and allocation. + */ +static int +ath12k_wifi7_dp_tx_encap_eapol(struct sk_buff *skb, + struct hal_tx_info *ti, + struct ath12k_skb_cb *skb_cb) +{ + struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; + enum hal_encrypt_type enc_type = + ath12k_dp_tx_get_encrypt_type(skb_cb->cipher); + u16 mac_hdr_len = ieee80211_hdrlen(hdr->frame_control); + u8 iv_len = ath12k_dp_tx_crypto_iv_len(enc_type); + u8 icv_len = ath12k_dp_tx_crypto_icv_len(enc_type); + + if (pskb_expand_head(skb, iv_len, icv_len, GFP_ATOMIC)) + return -ENOMEM; + + if (iv_len) { + skb_push(skb, iv_len); + memmove(skb->data, skb->data + iv_len, mac_hdr_len); + memset(skb->data + mac_hdr_len, 0, iv_len); + } + + if (icv_len) + memset(skb_put(skb, icv_len), 0, icv_len); + + ti->flags0 |= u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_TO_FW); + ti->encap_type = HAL_TCL_ENCAP_TYPE_RAW; + ti->encrypt_type = enc_type; + + return 0; +} + static void ath12k_wifi7_hal_tx_cmd_ext_desc_setup(struct ath12k_base *ab, struct hal_tx_msdu_ext_desc *tcl_ext_cmd, @@ -91,6 +134,7 @@ int ath12k_wifi7_dp_tx(struct ath12k_pdev_dp *dp_pdev, struct ath12k_link_vif *a u32 iova_mask = dp->hw_params->iova_mask; bool is_diff_encap = false; bool is_null_frame = false; + bool eapol_encap_done = false; if (test_bit(ATH12K_FLAG_CRASH_FLUSH, &ab->dev_flags)) return -ESHUTDOWN; @@ -211,9 +255,27 @@ tcl_ring_sel: case HAL_TCL_ENCAP_TYPE_NATIVE_WIFI: is_null_frame = ieee80211_is_nullfunc(hdr->frame_control); if (ahvif->vif->offload_flags & IEEE80211_OFFLOAD_ENCAP_ENABLED) { - if (skb->protocol == cpu_to_be16(ETH_P_PAE) || is_null_frame) + if ((skb->protocol == cpu_to_be16(ETH_P_PAE) && + !(skb_cb->flags & ATH12K_SKB_CIPHER_SET)) || is_null_frame) is_diff_encap = true; + if (skb->protocol == cpu_to_be16(ETH_P_PAE) && + (skb_cb->flags & ATH12K_SKB_CIPHER_SET)) { + if (!eapol_encap_done) { + ret = ath12k_wifi7_dp_tx_encap_eapol(skb, &ti, + skb_cb); + if (ret) + goto fail_remove_tx_buf; + hdr = (void *)skb->data; + eapol_encap_done = true; + } else { + ti.flags0 |= u32_encode_bits(1, + HAL_TCL_DATA_CMD_INFO2_TO_FW); + ti.encap_type = HAL_TCL_ENCAP_TYPE_RAW; + ti.encrypt_type = + ath12k_dp_tx_get_encrypt_type(skb_cb->cipher); + } + } /* Firmware expects msdu ext descriptor for nwifi/raw packets * received in ETH mode. Without this, observed tx fail for * Multicast packets in ETH mode. diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_rx_desc.h b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx_desc.h index 0d19a9cbb68ce9..6d69851e529d85 100644 --- a/drivers/net/wireless/ath/ath12k/wifi7/hal_rx_desc.h +++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx_desc.h @@ -140,6 +140,38 @@ struct rx_mpdu_start_qcn9274 { __le32 res1; } __packed; +struct rx_mpdu_start_qcc2072 { + __le32 info0; + __le32 info2; + __le32 reo_queue_desc_lo; + __le32 info1; + __le32 pn[4]; + __le32 info4; + __le32 peer_meta_data; + __le16 ast_index; + __le16 sw_peer_id; + __le16 info3; + __le16 phy_ppdu_id; + __le32 info5; + __le32 info6; + __le16 frame_ctrl; + __le16 duration; + u8 addr1[ETH_ALEN]; + u8 addr2[ETH_ALEN]; + u8 addr3[ETH_ALEN]; + __le16 seq_ctrl; + u8 addr4[ETH_ALEN]; + __le16 qos_ctrl; + __le32 ht_ctrl; + __le32 info7; + __le32 res0; + __le32 res1; + __le32 res2; + __le32 info8; + __le32 res3; + __le32 res4; +} __packed; + #define QCN9274_MPDU_START_SELECT_MPDU_START_TAG BIT(0) #define QCN9274_MPDU_START_SELECT_INFO0_REO_QUEUE_DESC_LO BIT(1) #define QCN9274_MPDU_START_SELECT_INFO1_PN_31_0 BIT(2) @@ -1492,7 +1524,7 @@ struct hal_rx_desc_qcc2072 { struct rx_msdu_end_qcn9274 msdu_end; u8 rx_padding0[RX_BE_PADDING0_BYTES]; __le32 mpdu_start_tag; - struct rx_mpdu_start_qcn9274 mpdu_start; + struct rx_mpdu_start_qcc2072 mpdu_start; struct rx_pkt_hdr_tlv_qcc2072 pkt_hdr_tlv; u8 msdu_payload[]; }; diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hw.c b/drivers/net/wireless/ath/ath12k/wifi7/hw.c index 03dedfd907fc7f..855bdfca34b11a 100644 --- a/drivers/net/wireless/ath/ath12k/wifi7/hw.c +++ b/drivers/net/wireless/ath/ath12k/wifi7/hw.c @@ -918,6 +918,7 @@ static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw, struct ethhdr *eth; bool is_prb_rsp; u16 mcbc_gsn; + u8 cb_flags; u8 link_id; int ret; struct ath12k_dp *tmp_dp; @@ -1011,8 +1012,13 @@ static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw, ieee80211_has_protected(hdr->frame_control)) is_dvlan = true; + /* + * Add a sta pointer check to differentiate multicast encapsulation + * offload packets, as the ATH12K_SKB_HW_80211_ENCAP flag is also set + * for such packets. + */ if (!vif->valid_links || !is_mcast || is_dvlan || - (skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) || + ((skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) && sta) || test_bit(ATH12K_FLAG_RAW_MODE, &ar->ab->dev_flags)) { ret = ath12k_wifi7_dp_tx(dp_pdev, arvif, arsta, skb, false, 0, is_mcast); if (unlikely(ret)) { @@ -1024,6 +1030,7 @@ static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw, mcbc_gsn = atomic_inc_return(&ahvif->dp_vif.mcbc_gsn) & 0xfff; links_map = ahvif->links_map; + cb_flags = skb_cb->flags; for_each_set_bit(link_id, &links_map, IEEE80211_MLD_MAX_NUM_LINKS) { tmp_arvif = rcu_dereference(ahvif->link[link_id]); @@ -1031,21 +1038,45 @@ static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw, continue; tmp_ar = tmp_arvif->ar; - tmp_dp_pdev = ath12k_dp_to_pdev_dp(tmp_ar->ab->dp, + tmp_dp = ath12k_ab_to_dp(tmp_ar->ab); + tmp_dp_pdev = ath12k_dp_to_pdev_dp(tmp_dp, tmp_ar->pdev_idx); if (!tmp_dp_pdev) continue; - msdu_copied = skb_copy(skb, GFP_ATOMIC); - if (!msdu_copied) { - ath12k_err(ar->ab, - "skb copy failure link_id 0x%X vdevid 0x%X\n", - link_id, tmp_arvif->vdev_id); - continue; - } - ath12k_mlo_mcast_update_tx_link_address(vif, link_id, - msdu_copied, - info_flags); + if (cb_flags & ATH12K_SKB_HW_80211_ENCAP) { + /* + * skb->data may be modified for the iova_mask devices. + * It is better to use skb_copy() for such devices + * to avoid any potential skb corruption related issues. + */ + if (tmp_dp->hw_params->iova_mask) + msdu_copied = skb_copy(skb, GFP_ATOMIC); + else + /* + * ath12k_wifi7_dp_tx() should treat cloned HW-encap + * Ethernet multicast frames as read-only. + */ + msdu_copied = skb_clone(skb, GFP_ATOMIC); + if (!msdu_copied) { + ath12k_err(ar->ab, + "skb copy/clone failure link_id 0x%X vdevid 0x%X\n", + link_id, tmp_arvif->vdev_id); + continue; + } + } else { + msdu_copied = skb_copy(skb, GFP_ATOMIC); + if (!msdu_copied) { + ath12k_err(ar->ab, + "skb copy failure link_id 0x%X vdevid 0x%X\n", + link_id, tmp_arvif->vdev_id); + continue; + } + + ath12k_mlo_mcast_update_tx_link_address(vif, link_id, + msdu_copied, + info_flags); + } skb_cb = ATH12K_SKB_CB(msdu_copied); skb_cb->link_id = link_id; @@ -1061,7 +1092,6 @@ static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw, if (unlikely(!ahvif->dp_vif.key_cipher)) goto skip_peer_find; - tmp_dp = ath12k_ab_to_dp(tmp_ar->ab); spin_lock_bh(&tmp_dp->dp_lock); peer = ath12k_dp_link_peer_find_by_addr(tmp_dp, tmp_arvif->bssid); @@ -1080,11 +1110,16 @@ static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw, skb_cb->cipher = key->cipher; skb_cb->flags |= ATH12K_SKB_CIPHER_SET; + if (skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) + goto skip_fctl_protected_check; + hdr = (struct ieee80211_hdr *)msdu_copied->data; if (!ieee80211_has_protected(hdr->frame_control)) hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_PROTECTED); } + +skip_fctl_protected_check: spin_unlock_bh(&tmp_dp->dp_lock); skip_peer_find: diff --git a/drivers/net/wireless/ath/ath12k/wmi.c b/drivers/net/wireless/ath/ath12k/wmi.c index 84a31b953db81a..90e1549ec43d7b 100644 --- a/drivers/net/wireless/ath/ath12k/wmi.c +++ b/drivers/net/wireless/ath/ath12k/wmi.c @@ -207,6 +207,8 @@ static const struct ath12k_wmi_tlv_policy ath12k_wmi_tlv_policies[] = { .min_len = sizeof(struct wmi_per_chain_rssi_stat_params) }, [WMI_TAG_OBSS_COLOR_COLLISION_EVT] = { .min_len = sizeof(struct wmi_obss_color_collision_event) }, + [WMI_TAG_PDEV_CSA_SWITCH_COUNT_STATUS_EVENT] = { + .min_len = sizeof(struct ath12k_wmi_pdev_csa_event) }, }; __le32 ath12k_wmi_tlv_hdr(u32 cmd, u32 len) @@ -374,6 +376,13 @@ ath12k_wmi_tlv_parse(struct ath12k_base *ab, struct sk_buff *skb) return tb; } +static u32 ath12k_wmi_tlv_data_len(const void *data) +{ + const struct wmi_tlv *tlv = (const struct wmi_tlv *)data - 1; + + return le32_get_bits(tlv->header, WMI_TLV_LEN); +} + static int ath12k_wmi_cmd_send_nowait(struct ath12k_wmi_pdev *wmi, struct sk_buff *skb, u32 cmd_id) { @@ -1228,10 +1237,16 @@ int ath12k_wmi_vdev_start(struct ath12k *ar, struct wmi_vdev_start_req_arg *arg, le32_encode_bits(arg->ml.mcast_link, ATH12K_WMI_FLAG_MLO_MCAST_VDEV) | le32_encode_bits(arg->ml.link_add, - ATH12K_WMI_FLAG_MLO_LINK_ADD); + ATH12K_WMI_FLAG_MLO_LINK_ADD) | + le32_encode_bits(arg->ml.assoc_link, + ATH12K_WMI_FLAG_MLO_START_AS_ACTIVE) | + cpu_to_le32(ATH12K_WMI_FLAG_MLO_IEEE_LINK_IDX_VALID); - ath12k_dbg(ar->ab, ATH12K_DBG_WMI, "vdev %d start ml flags 0x%x\n", - arg->vdev_id, ml_params->flags); + ml_params->ieee_link_id = cpu_to_le32(arg->ml.ieee_link_id); + + ath12k_dbg(ar->ab, ATH12K_DBG_WMI, "vdev %u start link_id %u ml flags 0x%x\n", + arg->vdev_id, arg->ml.ieee_link_id, + le32_to_cpu(ml_params->flags)); ptr += sizeof(*ml_params); @@ -1244,19 +1259,23 @@ int ath12k_wmi_vdev_start(struct ath12k *ar, struct wmi_vdev_start_req_arg *arg, partner_info = ptr; for (i = 0; i < arg->ml.num_partner_links; i++) { + struct wmi_ml_partner_info *pinfo = &arg->ml.partner_info[i]; + partner_info->tlv_header = ath12k_wmi_tlv_cmd_hdr(WMI_TAG_MLO_PARTNER_LINK_PARAMS, sizeof(*partner_info)); - partner_info->vdev_id = - cpu_to_le32(arg->ml.partner_info[i].vdev_id); - partner_info->hw_link_id = - cpu_to_le32(arg->ml.partner_info[i].hw_link_id); + partner_info->vdev_id = cpu_to_le32(pinfo->vdev_id); + partner_info->hw_link_id = cpu_to_le32(pinfo->hw_link_id); ether_addr_copy(partner_info->vdev_addr.addr, - arg->ml.partner_info[i].addr); + pinfo->addr); + partner_info->flags = + cpu_to_le32(ATH12K_WMI_FLAG_MLO_IEEE_LINK_IDX_VALID_PARTNER); + partner_info->ieee_link_id = cpu_to_le32(pinfo->ieee_link_id); - ath12k_dbg(ar->ab, ATH12K_DBG_WMI, "partner vdev %d hw_link_id %d macaddr%pM\n", - partner_info->vdev_id, partner_info->hw_link_id, - partner_info->vdev_addr.addr); + ath12k_dbg(ar->ab, ATH12K_DBG_WMI, "partner vdev %u hw_link_id %u macaddr %pM link_id %u ml flags 0x%x\n", + pinfo->vdev_id, pinfo->hw_link_id, + pinfo->addr, pinfo->ieee_link_id, + le32_to_cpu(partner_info->flags)); partner_info++; } @@ -2788,8 +2807,8 @@ int ath12k_wmi_send_scan_start_cmd(struct ath12k *ar, for (i = 0; i < arg->num_hint_bssid; ++i) { hint_bssid->freq_flags = arg->hint_bssid[i].freq_flags; - ether_addr_copy(&arg->hint_bssid[i].bssid.addr[0], - &hint_bssid->bssid.addr[0]); + ether_addr_copy(&hint_bssid->bssid.addr[0], + &arg->hint_bssid[i].bssid.addr[0]); hint_bssid++; } } @@ -4751,14 +4770,16 @@ static int ath12k_wmi_mac_phy_caps_parse(struct ath12k_base *soc, if (svc_rdy_ext->n_mac_phy_caps >= svc_rdy_ext->tot_phy_id) return -ENOBUFS; - len = min_t(u16, len, sizeof(struct ath12k_wmi_mac_phy_caps_params)); if (!svc_rdy_ext->n_mac_phy_caps) { - svc_rdy_ext->mac_phy_caps = kzalloc((svc_rdy_ext->tot_phy_id) * len, - GFP_ATOMIC); + svc_rdy_ext->mac_phy_caps = + kzalloc_objs(*svc_rdy_ext->mac_phy_caps, + svc_rdy_ext->tot_phy_id, + GFP_ATOMIC); if (!svc_rdy_ext->mac_phy_caps) return -ENOMEM; } + len = min_t(u16, len, sizeof(struct ath12k_wmi_mac_phy_caps_params)); memcpy(svc_rdy_ext->mac_phy_caps + svc_rdy_ext->n_mac_phy_caps, ptr, len); svc_rdy_ext->n_mac_phy_caps++; return 0; @@ -6713,16 +6734,12 @@ static int ath12k_pull_roam_ev(struct ath12k_base *ab, struct sk_buff *skb, return 0; } -static int freq_to_idx(struct ath12k *ar, int freq) +static int freq_to_idx(struct ieee80211_hw *hw, int freq) { struct ieee80211_supported_band *sband; - struct ieee80211_hw *hw = ath12k_ar_to_hw(ar); int band, ch, idx = 0; for (band = NL80211_BAND_2GHZ; band < NUM_NL80211_BANDS; band++) { - if (!ar->mac.sbands[band].channels) - continue; - sband = hw->wiphy->bands[band]; if (!sband) continue; @@ -7072,25 +7089,29 @@ static void ath12k_peer_delete_resp_event(struct ath12k_base *ab, struct sk_buff { struct wmi_peer_delete_resp_event peer_del_resp; struct ath12k *ar; + u32 vdev_id; if (ath12k_pull_peer_del_resp_ev(ab, skb, &peer_del_resp) != 0) { - ath12k_warn(ab, "failed to extract peer delete resp"); + ath12k_warn(ab, "failed to extract peer delete resp\n"); return; } + vdev_id = le32_to_cpu(peer_del_resp.vdev_id); + rcu_read_lock(); - ar = ath12k_mac_get_ar_by_vdev_id(ab, le32_to_cpu(peer_del_resp.vdev_id)); + ar = ath12k_mac_get_ar_by_vdev_id(ab, vdev_id); if (!ar) { - ath12k_warn(ab, "invalid vdev id in peer delete resp ev %d", - peer_del_resp.vdev_id); + ath12k_warn(ab, "invalid vdev id in peer delete resp ev %d\n", + vdev_id); rcu_read_unlock(); return; } - complete(&ar->peer_delete_done); + ath12k_peer_delete_resp_signal(ar, vdev_id, + peer_del_resp.peer_macaddr.addr); rcu_read_unlock(); ath12k_dbg(ab, ATH12K_DBG_WMI, "peer delete resp for vdev id %d addr %pM\n", - peer_del_resp.vdev_id, peer_del_resp.peer_macaddr.addr); + vdev_id, peer_del_resp.peer_macaddr.addr); } static void ath12k_vdev_delete_resp_event(struct ath12k_base *ab, @@ -7629,6 +7650,7 @@ static void ath12k_chan_info_event(struct ath12k_base *ab, struct sk_buff *skb) { struct wmi_chan_info_event ch_info_ev = {}; struct ath12k *ar; + struct ath12k_hw *ah; struct survey_info *survey; int idx; /* HW channel counters frequency value in hertz */ @@ -7660,6 +7682,7 @@ static void ath12k_chan_info_event(struct ath12k_base *ab, struct sk_buff *skb) return; } spin_lock_bh(&ar->data_lock); + ah = ath12k_ar_to_ah(ar); switch (ar->scan.state) { case ATH12K_SCAN_IDLE: @@ -7671,8 +7694,8 @@ static void ath12k_chan_info_event(struct ath12k_base *ab, struct sk_buff *skb) break; } - idx = freq_to_idx(ar, le32_to_cpu(ch_info_ev.freq)); - if (idx >= ARRAY_SIZE(ar->survey)) { + idx = freq_to_idx(ath12k_ar_to_hw(ar), le32_to_cpu(ch_info_ev.freq)); + if (idx >= ARRAY_SIZE(ah->survey)) { ath12k_warn(ab, "chan info: invalid frequency %d (idx %d out of bounds)\n", ch_info_ev.freq, idx); goto exit; @@ -7685,14 +7708,20 @@ static void ath12k_chan_info_event(struct ath12k_base *ab, struct sk_buff *skb) cc_freq_hz = (le32_to_cpu(ch_info_ev.mac_clk_mhz) * 1000); if (ch_info_ev.cmd_flags == WMI_CHAN_INFO_START_RESP) { - survey = &ar->survey[idx]; - memset(survey, 0, sizeof(*survey)); - survey->noise = le32_to_cpu(ch_info_ev.noise_floor); - survey->filled = SURVEY_INFO_NOISE_DBM | SURVEY_INFO_TIME | - SURVEY_INFO_TIME_BUSY; - survey->time = div_u64(le32_to_cpu(ch_info_ev.cycle_count), cc_freq_hz); - survey->time_busy = div_u64(le32_to_cpu(ch_info_ev.rx_clear_count), - cc_freq_hz); + scoped_guard(spinlock_bh, &ah->survey_lock) { + survey = &ah->survey[idx]; + memset(survey, 0, sizeof(*survey)); + survey->noise = le32_to_cpu(ch_info_ev.noise_floor); + survey->time = + div_u64(le32_to_cpu(ch_info_ev.cycle_count), + cc_freq_hz); + survey->time_busy = + div_u64(le32_to_cpu(ch_info_ev.rx_clear_count), + cc_freq_hz); + survey->filled = SURVEY_INFO_NOISE_DBM | + SURVEY_INFO_TIME | + SURVEY_INFO_TIME_BUSY; + } } exit: spin_unlock_bh(&ar->data_lock); @@ -7705,6 +7734,7 @@ ath12k_pdev_bss_chan_info_event(struct ath12k_base *ab, struct sk_buff *skb) struct wmi_pdev_bss_chan_info_event bss_ch_info_ev = {}; struct survey_info *survey; struct ath12k *ar; + struct ath12k_hw *ah; u32 cc_freq_hz = ab->cc_freq_hz; u64 busy, total, tx, rx, rx_bss; int idx; @@ -7745,28 +7775,31 @@ ath12k_pdev_bss_chan_info_event(struct ath12k_base *ab, struct sk_buff *skb) return; } - spin_lock_bh(&ar->data_lock); - idx = freq_to_idx(ar, le32_to_cpu(bss_ch_info_ev.freq)); - if (idx >= ARRAY_SIZE(ar->survey)) { + ah = ath12k_ar_to_ah(ar); + + idx = freq_to_idx(ath12k_ar_to_hw(ar), le32_to_cpu(bss_ch_info_ev.freq)); + if (idx >= ARRAY_SIZE(ah->survey)) { ath12k_warn(ab, "bss chan info: invalid frequency %d (idx %d out of bounds)\n", bss_ch_info_ev.freq, idx); goto exit; } - survey = &ar->survey[idx]; + scoped_guard(spinlock_bh, &ah->survey_lock) { + survey = &ah->survey[idx]; + + survey->noise = le32_to_cpu(bss_ch_info_ev.noise_floor); + survey->time = div_u64(total, cc_freq_hz); + survey->time_busy = div_u64(busy, cc_freq_hz); + survey->time_rx = div_u64(rx_bss, cc_freq_hz); + survey->time_tx = div_u64(tx, cc_freq_hz); + survey->filled |= (SURVEY_INFO_NOISE_DBM | + SURVEY_INFO_TIME | + SURVEY_INFO_TIME_BUSY | + SURVEY_INFO_TIME_RX | + SURVEY_INFO_TIME_TX); + } - survey->noise = le32_to_cpu(bss_ch_info_ev.noise_floor); - survey->time = div_u64(total, cc_freq_hz); - survey->time_busy = div_u64(busy, cc_freq_hz); - survey->time_rx = div_u64(rx_bss, cc_freq_hz); - survey->time_tx = div_u64(tx, cc_freq_hz); - survey->filled |= (SURVEY_INFO_NOISE_DBM | - SURVEY_INFO_TIME | - SURVEY_INFO_TIME_BUSY | - SURVEY_INFO_TIME_RX | - SURVEY_INFO_TIME_TX); exit: - spin_unlock_bh(&ar->data_lock); complete(&ar->bss_survey_done); rcu_read_unlock(); @@ -9034,12 +9067,19 @@ ath12k_wmi_process_csa_switch_count_event(struct ath12k_base *ab, const u32 *vdev_ids) { u32 current_switch_count = le32_to_cpu(ev->current_switch_count); + u32 vdev_ids_len = ath12k_wmi_tlv_data_len(vdev_ids); u32 num_vdevs = le32_to_cpu(ev->num_vdevs); struct ieee80211_bss_conf *conf; struct ath12k_link_vif *arvif; struct ath12k_vif *ahvif; int i; + if (num_vdevs > vdev_ids_len / sizeof(*vdev_ids)) { + ath12k_warn(ab, "csa switch count num_vdevs %u exceeds tlv array length %u\n", + num_vdevs, vdev_ids_len); + return; + } + rcu_read_lock(); for (i = 0; i < num_vdevs; i++) { arvif = ath12k_mac_get_arvif_by_vdev_id(ab, vdev_ids[i]); @@ -9929,6 +9969,7 @@ static void ath12k_wmi_process_tpc_stats(struct ath12k_base *ab, void *ptr = skb->data; struct ath12k *ar; u16 tlv_tag; + u16 tlv_len; u32 event_count; int ret; @@ -9944,6 +9985,7 @@ static void ath12k_wmi_process_tpc_stats(struct ath12k_base *ab, tlv = (struct wmi_tlv *)ptr; tlv_tag = le32_get_bits(tlv->header, WMI_TLV_TAG); + tlv_len = le32_get_bits(tlv->header, WMI_TLV_LEN); ptr += sizeof(*tlv); if (tlv_tag != WMI_TAG_HALPHY_CTRL_PATH_EVENT_FIXED_PARAM) { @@ -9951,6 +9993,12 @@ static void ath12k_wmi_process_tpc_stats(struct ath12k_base *ab, return; } + if (tlv_len < sizeof(*fixed_param)) { + ath12k_warn(ab, "TPC stats fixed param tlv len %u too short\n", + tlv_len); + return; + } + fixed_param = (struct ath12k_wmi_pdev_tpc_stats_event_fixed_params *)ptr; rcu_read_lock(); ar = ath12k_mac_get_ar_by_pdev_id(ab, le32_to_cpu(fixed_param->pdev_id) + 1); @@ -10257,12 +10305,12 @@ static void ath12k_wmi_op_rx(struct ath12k_base *ab, struct sk_buff *skb) struct wmi_cmd_hdr *cmd_hdr; enum wmi_tlv_event_id id; - cmd_hdr = (struct wmi_cmd_hdr *)skb->data; - id = le32_get_bits(cmd_hdr->cmd_id, WMI_CMD_HDR_CMD_ID); - - if (!skb_pull(skb, sizeof(struct wmi_cmd_hdr))) + cmd_hdr = skb_pull_data(skb, sizeof(*cmd_hdr)); + if (!cmd_hdr) goto out; + id = le32_get_bits(cmd_hdr->cmd_id, WMI_CMD_HDR_CMD_ID); + switch (id) { /* Process all the WMI events here */ case WMI_SERVICE_READY_EVENTID: diff --git a/drivers/net/wireless/ath/ath12k/wmi.h b/drivers/net/wireless/ath/ath12k/wmi.h index c452e3d57a29ad..20e3939e8820bb 100644 --- a/drivers/net/wireless/ath/ath12k/wmi.h +++ b/drivers/net/wireless/ath/ath12k/wmi.h @@ -2954,10 +2954,14 @@ struct wmi_vdev_create_mlo_params { #define ATH12K_WMI_FLAG_MLO_EMLSR_SUPPORT BIT(6) #define ATH12K_WMI_FLAG_MLO_FORCED_INACTIVE BIT(7) #define ATH12K_WMI_FLAG_MLO_LINK_ADD BIT(8) +#define ATH12K_WMI_FLAG_MLO_START_AS_ACTIVE BIT(17) +#define ATH12K_WMI_FLAG_MLO_IEEE_LINK_IDX_VALID BIT(18) +#define ATH12K_WMI_FLAG_MLO_IEEE_LINK_IDX_VALID_PARTNER BIT(19) struct wmi_vdev_start_mlo_params { __le32 tlv_header; __le32 flags; + __le32 ieee_link_id; } __packed; struct wmi_partner_link_info { @@ -2965,6 +2969,8 @@ struct wmi_partner_link_info { __le32 vdev_id; __le32 hw_link_id; struct ath12k_wmi_mac_addr_params vdev_addr; + __le32 flags; + __le32 ieee_link_id; } __packed; struct wmi_vdev_delete_cmd { @@ -3120,6 +3126,7 @@ struct wmi_ml_partner_info { bool primary_umac; bool logical_link_idx_valid; u32 logical_link_idx; + u32 ieee_link_id; }; struct wmi_ml_arg { @@ -3127,6 +3134,7 @@ struct wmi_ml_arg { bool assoc_link; bool mcast_link; bool link_add; + u32 ieee_link_id; u8 num_partner_links; struct wmi_ml_partner_info partner_info[ATH12K_WMI_MLO_MAX_LINKS]; }; diff --git a/drivers/net/wireless/ath/ath6kl/wmi.c b/drivers/net/wireless/ath/ath6kl/wmi.c index 2b0c5038ae0403..6c29f0bcec9f55 100644 --- a/drivers/net/wireless/ath/ath6kl/wmi.c +++ b/drivers/net/wireless/ath/ath6kl/wmi.c @@ -1296,6 +1296,9 @@ static int ath6kl_wmi_scan_complete_rx(struct wmi *wmi, u8 *datap, int len, { struct wmi_scan_complete_event *ev; + if (len < sizeof(*ev)) + return -EINVAL; + ev = (struct wmi_scan_complete_event *) datap; ath6kl_scan_complete_evt(vif, a_sle32_to_cpu(ev->status)); @@ -3372,7 +3375,12 @@ static int ath6kl_wmi_get_pmkid_list_event_rx(struct wmi *wmi, u8 *datap, static int ath6kl_wmi_addba_req_event_rx(struct wmi *wmi, u8 *datap, int len, struct ath6kl_vif *vif) { - struct wmi_addba_req_event *cmd = (struct wmi_addba_req_event *) datap; + struct wmi_addba_req_event *cmd; + + if (len < sizeof(*cmd)) + return -EINVAL; + + cmd = (struct wmi_addba_req_event *)datap; aggr_recv_addba_req_evt(vif, cmd->tid, le16_to_cpu(cmd->st_seq_no), cmd->win_sz); @@ -3383,7 +3391,12 @@ static int ath6kl_wmi_addba_req_event_rx(struct wmi *wmi, u8 *datap, int len, static int ath6kl_wmi_delba_req_event_rx(struct wmi *wmi, u8 *datap, int len, struct ath6kl_vif *vif) { - struct wmi_delba_event *cmd = (struct wmi_delba_event *) datap; + struct wmi_delba_event *cmd; + + if (len < sizeof(*cmd)) + return -EINVAL; + + cmd = (struct wmi_delba_event *)datap; aggr_recv_delba_req_evt(vif, cmd->tid); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index 92c16a3173288b..66557be28e7b16 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -1283,6 +1283,9 @@ static s32 brcmf_p2p_abort_action_frame(struct brcmf_cfg80211_info *cfg) brcmf_dbg(TRACE, "Enter\n"); vif = p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif; + if (!vif) + vif = p2p->bss_idx[P2PAPI_BSSCFG_PRIMARY].vif; + err = brcmf_fil_bsscfg_data_set(vif->ifp, "actframe_abort", &int_val, sizeof(s32)); if (err) @@ -1819,6 +1822,7 @@ bool brcmf_p2p_send_action_frame(struct brcmf_if *ifp, /* validate channel and p2p ies */ if (config_af_params.search_channel && IS_P2P_SOCIAL_CHANNEL(le32_to_cpu(af_params->channel)) && + p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif && p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif->saved_ie.probe_req_ie_len) { afx_hdl = &p2p->afx_hdl; afx_hdl->peer_listen_chan = le32_to_cpu(af_params->channel); diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dump.c b/drivers/net/wireless/intel/iwlwifi/fw/dump.c index c2af66899a780b..bbbf3669a555d9 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/dump.c +++ b/drivers/net/wireless/intel/iwlwifi/fw/dump.c @@ -369,7 +369,7 @@ static void iwl_fwrt_dump_fseq_regs(struct iwl_fw_runtime *fwrt) void iwl_fwrt_dump_error_logs(struct iwl_fw_runtime *fwrt) { struct iwl_pc_data *pc_data; - u8 count; + u32 count; if (!iwl_trans_device_enabled(fwrt->trans)) { IWL_ERR(fwrt, diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c index b1a55909f0d4bc..11763dec77eca6 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2018-2025 Intel Corporation + * Copyright (C) 2018-2026 Intel Corporation */ #include #include "iwl-drv.h" @@ -611,6 +611,9 @@ static int iwl_dbg_tlv_alloc_fragments(struct iwl_fw_runtime *fwrt, cpu_to_le32(IWL_FW_INI_LOCATION_DRAM_PATH)) return 0; + if (!fw_mon_cfg->req_size) + return -EIO; + num_frags = le32_to_cpu(fw_mon_cfg->max_frags_num); if (fwrt->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210) { if (alloc_id != IWL_FW_INI_ALLOCATION_ID_DBGC1) @@ -621,6 +624,9 @@ static int iwl_dbg_tlv_alloc_fragments(struct iwl_fw_runtime *fwrt, return -EIO; } + if (!num_frags) + return -EIO; + remain_pages = DIV_ROUND_UP(le32_to_cpu(fw_mon_cfg->req_size), PAGE_SIZE); num_frags = min_t(u32, num_frags, BUF_ALLOC_MAX_NUM_FRAGS); diff --git a/drivers/net/wireless/intel/iwlwifi/mei/main.c b/drivers/net/wireless/intel/iwlwifi/mei/main.c index c5ff1b1b720f36..b78ab75afc1593 100644 --- a/drivers/net/wireless/intel/iwlwifi/mei/main.c +++ b/drivers/net/wireless/intel/iwlwifi/mei/main.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2021-2024 Intel Corporation + * Copyright (C) 2026 Intel Corporation */ #include @@ -457,7 +458,7 @@ static int iwl_mei_send_sap_msg_payload(struct mei_cl_device *cldev, notif_q = &dir->q_ctrl_blk[SAP_QUEUE_IDX_NOTIF]; q_head = mei->shared_mem.q_head[SAP_DIRECTION_HOST_TO_ME][SAP_QUEUE_IDX_NOTIF]; q_sz = mei->shared_mem.q_size[SAP_DIRECTION_HOST_TO_ME][SAP_QUEUE_IDX_NOTIF]; - ret = iwl_mei_write_cyclic_buf(q_head, notif_q, q_head, hdr, q_sz); + ret = iwl_mei_write_cyclic_buf(cldev, notif_q, q_head, hdr, q_sz); if (ret < 0) return ret; @@ -1147,6 +1148,11 @@ static void iwl_mei_handle_sap_rx_cmd(struct mei_cl_device *cldev, iwl_mei_read_from_q(q_head, q_sz, &rd, wr, hdr, sizeof(*hdr)); valid_rx_sz -= sizeof(*hdr); len = le16_to_cpu(hdr->len); + if (len + sizeof(*hdr) > PAGE_SIZE) { + dev_err(&cldev->dev, + "SAP message is too big: %u\n", len); + break; + } if (valid_rx_sz < len) break; diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c index 2297392db9558b..b07a78524bcc8c 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c @@ -954,7 +954,7 @@ static void iwl_mvm_frob_txf_key_iter(struct ieee80211_hw *hw, } match++; if (match == keylen) { - memset(txf->buf + i - keylen, 0xAA, keylen); + memset(txf->buf + i + 1 - keylen, 0xAA, keylen); match = 0; } } diff --git a/drivers/net/wireless/mediatek/mt76/channel.c b/drivers/net/wireless/mediatek/mt76/channel.c index 6edcb3b8f2798e..28ad7bcaffd459 100644 --- a/drivers/net/wireless/mediatek/mt76/channel.c +++ b/drivers/net/wireless/mediatek/mt76/channel.c @@ -186,68 +186,74 @@ int mt76_switch_vif_chanctx(struct ieee80211_hw *hw, int n_vifs, enum ieee80211_chanctx_switch_mode mode) { - struct mt76_chanctx *old_ctx = (struct mt76_chanctx *)vifs->old_ctx->drv_priv; - struct mt76_chanctx *new_ctx = (struct mt76_chanctx *)vifs->new_ctx->drv_priv; - struct ieee80211_chanctx_conf *conf = vifs->new_ctx; - struct mt76_phy *old_phy = old_ctx->phy; - struct mt76_phy *phy = hw->priv; + struct ieee80211_vif_chanctx_switch *v; + struct mt76_chanctx *old_ctx, *new_ctx; + struct mt76_phy *old_phy, *phy = hw->priv; struct mt76_dev *dev = phy->dev; struct mt76_vif_link *mlink; - bool update_chan; + bool need_update[__MT_MAX_BAND] = {}; int i, ret = 0; - if (mode == CHANCTX_SWMODE_SWAP_CONTEXTS) - phy = new_ctx->phy = dev->band_phys[conf->def.chan->band]; - else - phy = new_ctx->phy; - if (!phy) - return -EINVAL; + for (i = 0; i < n_vifs; i++) { + v = &vifs[i]; + new_ctx = (struct mt76_chanctx *)v->new_ctx->drv_priv; + if (mode == CHANCTX_SWMODE_SWAP_CONTEXTS) + phy = new_ctx->phy = dev->band_phys[v->new_ctx->def.chan->band]; + else + phy = new_ctx->phy; - update_chan = phy->chanctx != new_ctx; - if (update_chan) { - if (dev->scan.phy == phy) - mt76_abort_scan(dev); + if (!phy) + return -EINVAL; - cancel_delayed_work_sync(&phy->mac_work); + if (need_update[phy->band_idx]) + continue; + + if (phy->chanctx != new_ctx) { + if (dev->scan.phy == phy) + mt76_abort_scan(dev); + + cancel_delayed_work_sync(&phy->mac_work); + need_update[phy->band_idx] = true; + } } mutex_lock(&dev->mutex); - if (mode == CHANCTX_SWMODE_SWAP_CONTEXTS && - phy != old_phy && old_phy->chanctx == old_ctx) - old_phy->chanctx = NULL; + for (i = 0; i < n_vifs; i++) { + v = &vifs[i]; + old_ctx = (struct mt76_chanctx *)v->old_ctx->drv_priv; + old_phy = old_ctx->phy; + + new_ctx = (struct mt76_chanctx *)v->new_ctx->drv_priv; + phy = new_ctx->phy; - if (update_chan) - ret = mt76_phy_update_channel(phy, vifs->new_ctx); + if (mode == CHANCTX_SWMODE_SWAP_CONTEXTS && old_phy->chanctx && + old_phy->chanctx == old_ctx && phy != old_phy) + old_phy->chanctx = NULL; - if (ret) - goto out; + if (need_update[phy->band_idx]) { + ret = mt76_phy_update_channel(phy, v->new_ctx); + if (ret) + goto out; - if (old_phy == phy) - goto skip_link_replace; + need_update[phy->band_idx] = false; + } - for (i = 0; i < n_vifs; i++) { - mlink = mt76_vif_conf_link(dev, vifs[i].vif, vifs[i].link_conf); + mlink = mt76_vif_conf_link(dev, v->vif, v->link_conf); if (!mlink) continue; - dev->drv->vif_link_remove(old_phy, vifs[i].vif, - vifs[i].link_conf, mlink); - - ret = dev->drv->vif_link_add(phy, vifs[i].vif, - vifs[i].link_conf, mlink); - if (ret) - goto out; - - } + if (old_phy != phy) { + dev->drv->vif_link_remove(old_phy, v->vif, v->link_conf, + mlink); -skip_link_replace: - for (i = 0; i < n_vifs; i++) { - mlink = mt76_vif_conf_link(dev, vifs[i].vif, vifs[i].link_conf); - if (!mlink) - continue; + ret = dev->drv->vif_link_add(phy, v->vif, v->link_conf, + mlink); + if (ret) + goto out; + } - mlink->ctx = vifs->new_ctx; + mlink->ctx = v->new_ctx; if (mlink->beacon_mon_interval) WRITE_ONCE(mlink->beacon_mon_last, jiffies); } diff --git a/drivers/net/wireless/mediatek/mt76/dma.c b/drivers/net/wireless/mediatek/mt76/dma.c index f8c2fe5f2f5877..a67880a9281c53 100644 --- a/drivers/net/wireless/mediatek/mt76/dma.c +++ b/drivers/net/wireless/mediatek/mt76/dma.c @@ -186,6 +186,18 @@ mt76_dma_queue_magic_cnt_init(struct mt76_dev *dev, struct mt76_queue *q) } } +/* A hung bus (e.g. after a PCIe AER error) reads 0xffffffff from every + * register, so clamp an out-of-range index to the fallback to keep it from + * corrupting q->head/q->tail. + */ +static int +mt76_dma_read_dma_idx(struct mt76_queue *q, int fallback) +{ + u32 idx = Q_READ(q, dma_idx); + + return idx < q->ndesc ? idx : fallback; +} + static void mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q) { @@ -201,7 +213,8 @@ mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q) } Q_WRITE(q, desc_base, q->desc_dma); - q->head = Q_READ(q, dma_idx); + + q->head = mt76_dma_read_dma_idx(q, 0); q->tail = q->head; } @@ -419,7 +432,7 @@ mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush) if (flush) last = -1; else - last = Q_READ(q, dma_idx); + last = mt76_dma_read_dma_idx(q, -1); while (q->queued > 0 && q->tail != last) { mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry); @@ -432,7 +445,7 @@ mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush) } if (!flush && q->tail == last) - last = Q_READ(q, dma_idx); + last = mt76_dma_read_dma_idx(q, -1); } spin_unlock_bh(&q->cleanup_lock); @@ -534,8 +547,13 @@ mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx, t->ptr = NULL; mt76_put_rxwi(dev, t); - if (drop) +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + /* the WO MCU owns the RX path only on WED v2, on newer + * versions this buf1 bit carries no drop information + */ + if (drop && dev->mmio.wed.version == 2) *drop |= !!(buf1 & MT_DMA_CTL_WO_DROP); +#endif } else { dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0], SKB_WITH_OVERHEAD(q->buf_size), @@ -595,6 +613,15 @@ mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush, q->desc[idx].ctrl |= cpu_to_le32(MT_DMA_CTL_DMA_DONE); else if (!(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE))) return NULL; +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + /* on WED v3 the M_DONE bit signals that WED is done reading + * the txfree descriptor; WED v2 does not set it + */ + else if (dev->mmio.wed.version > 2 && + mt76_queue_is_wed_tx_free(q) && + !(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_M_DONE))) + return NULL; +#endif } done: q->tail = (q->tail + 1) % q->ndesc; @@ -625,8 +652,8 @@ mt76_dma_tx_queue_skb_raw(struct mt76_dev *dev, struct mt76_queue *q, buf.len = skb->len; spin_lock_bh(&q->lock); - mt76_dma_add_buf(dev, q, &buf, 1, tx_info, skb, NULL); - mt76_dma_kick_queue(dev, q); + if (mt76_dma_add_buf(dev, q, &buf, 1, tx_info, skb, NULL) >= 0) + mt76_dma_kick_queue(dev, q); spin_unlock_bh(&q->lock); return 0; @@ -977,23 +1004,24 @@ mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget) struct sk_buff *skb; unsigned char *data; bool check_ddone = false; - bool allow_direct = !mt76_queue_is_wed_rx(q); + bool allow_direct = !mt76_queue_is_wed_rx(q) && + !mt76_queue_is_wed_rro_rxdmad_c(q); bool more; if ((q->flags & MT_QFLAG_WED_RRO_EN) || (IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED) && mt76_queue_is_wed_tx_free(q))) { - dma_idx = Q_READ(q, dma_idx); + dma_idx = mt76_dma_read_dma_idx(q, q->tail); check_ddone = true; } while (done < budget) { bool drop = false; - u32 info; + u32 info = 0; if (check_ddone) { if (q->tail == dma_idx) - dma_idx = Q_READ(q, dma_idx); + dma_idx = mt76_dma_read_dma_idx(q, q->tail); if (q->tail == dma_idx) break; diff --git a/drivers/net/wireless/mediatek/mt76/dma.h b/drivers/net/wireless/mediatek/mt76/dma.h index 2a0226c83f3c92..a2cf82cfdaaaa9 100644 --- a/drivers/net/wireless/mediatek/mt76/dma.h +++ b/drivers/net/wireless/mediatek/mt76/dma.h @@ -11,7 +11,7 @@ #define MT_DMA_CTL_SD_LEN1 GENMASK(13, 0) #define MT_DMA_CTL_LAST_SEC1 BIT(14) -#define MT_DMA_CTL_BURST BIT(15) +#define MT_DMA_CTL_M_DONE BIT(15) #define MT_DMA_CTL_SD_LEN0 GENMASK(29, 16) #define MT_DMA_CTL_LAST_SEC0 BIT(30) #define MT_DMA_CTL_DMA_DONE BIT(31) diff --git a/drivers/net/wireless/mediatek/mt76/mac80211.c b/drivers/net/wireless/mediatek/mt76/mac80211.c index 13c4e8abe28191..f92277770488aa 100644 --- a/drivers/net/wireless/mediatek/mt76/mac80211.c +++ b/drivers/net/wireless/mediatek/mt76/mac80211.c @@ -681,6 +681,7 @@ mt76_alloc_device(struct device *pdev, unsigned int size, dev = hw->priv; dev->hw = hw; dev->dev = pdev; + dev->init_wiphy = NULL; dev->drv = drv_ops; dev->dma_dev = pdev; @@ -779,6 +780,12 @@ int mt76_register_device(struct mt76_dev *dev, bool vht, mt76_check_sband(&dev->phy, &phy->sband_5g, NL80211_BAND_5GHZ); mt76_check_sband(&dev->phy, &phy->sband_6g, NL80211_BAND_6GHZ); + if (dev->init_wiphy) { + ret = dev->init_wiphy(dev); + if (ret) + return ret; + } + if (IS_ENABLED(CONFIG_MT76_LEDS)) { ret = mt76_led_init(phy); if (ret) @@ -886,6 +893,7 @@ static void mt76_rx_release_amsdu(struct mt76_phy *phy, enum mt76_rxq_id q) struct sk_buff *skb = phy->rx_amsdu[q].head; struct mt76_rx_status *status = (struct mt76_rx_status *)skb->cb; struct mt76_dev *dev = phy->dev; + struct mt76_queue *rxq = &dev->q_rx[q]; phy->rx_amsdu[q].head = NULL; phy->rx_amsdu[q].tail = NULL; @@ -914,6 +922,13 @@ static void mt76_rx_release_amsdu(struct mt76_phy *phy, enum mt76_rxq_id q) return; } } + + /* RRO 3.0 data queue skbs are processed and completed in the context + * of the indicator queue NAPI, which only polls its own skb list + */ + if (mt76_queue_is_wed_rro_data(rxq) && dev->hwrro_mode == MT76_HWRRO_V3) + q = MT_RXQ_RRO_IND; + __skb_queue_tail(&dev->rx_skb[q], skb); } diff --git a/drivers/net/wireless/mediatek/mt76/mcu.c b/drivers/net/wireless/mediatek/mt76/mcu.c index cbfb3bbec5031e..7149b2f7aafdee 100644 --- a/drivers/net/wireless/mediatek/mt76/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mcu.c @@ -78,15 +78,19 @@ int mt76_mcu_skb_send_and_get_msg(struct mt76_dev *dev, struct sk_buff *skb, unsigned long expires; int ret, seq; - if (mt76_is_sdio(dev)) - if (test_bit(MT76_RESET, &dev->phy.state) && atomic_read(&dev->bus_hung)) - return -EIO; - if (ret_skb) *ret_skb = NULL; mutex_lock(&dev->mcu.mutex); + if ((mt76_is_mmio(dev) && atomic_read(&dev->bus_hung)) || + (mt76_is_sdio(dev) && test_bit(MT76_RESET, &dev->phy.state) && + atomic_read(&dev->bus_hung))) { + orig_skb = skb; + ret = -EIO; + goto out; + } + if (dev->mcu_ops->mcu_skb_prepare_msg) { orig_skb = skb; ret = dev->mcu_ops->mcu_skb_prepare_msg(dev, skb, cmd, &seq); diff --git a/drivers/net/wireless/mediatek/mt76/mmio.c b/drivers/net/wireless/mediatek/mt76/mmio.c index 05d74cd7248e97..73d47608bf428d 100644 --- a/drivers/net/wireless/mediatek/mt76/mmio.c +++ b/drivers/net/wireless/mediatek/mt76/mmio.c @@ -35,9 +35,16 @@ static void mt76_mmio_write_copy(struct mt76_dev *dev, u32 offset, { int i; - for (i = 0; i < ALIGN(len, 4); i += 4) + for (i = 0; i + 4 <= len; i += 4) writel(get_unaligned_le32(data + i), dev->mmio.regs + offset + i); + + if (i < len) { + u8 tmp[4] = {}; + + memcpy(tmp, data + i, len - i); + writel(get_unaligned_le32(tmp), dev->mmio.regs + offset + i); + } } static void mt76_mmio_read_copy(struct mt76_dev *dev, u32 offset, @@ -45,9 +52,16 @@ static void mt76_mmio_read_copy(struct mt76_dev *dev, u32 offset, { int i; - for (i = 0; i < ALIGN(len, 4); i += 4) + for (i = 0; i + 4 <= len; i += 4) put_unaligned_le32(readl(dev->mmio.regs + offset + i), data + i); + + if (i < len) { + u8 tmp[4]; + + put_unaligned_le32(readl(dev->mmio.regs + offset + i), tmp); + memcpy(data + i, tmp, len - i); + } } static int mt76_mmio_wr_rp(struct mt76_dev *dev, u32 base, diff --git a/drivers/net/wireless/mediatek/mt76/mt76.h b/drivers/net/wireless/mediatek/mt76/mt76.h index 3822eb8fd88fc2..476578e2fbf0c5 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76.h +++ b/drivers/net/wireless/mediatek/mt76/mt76.h @@ -672,6 +672,7 @@ struct mt76_usb { u8 out_ep[__MT_EP_OUT_MAX]; u8 in_ep[__MT_EP_IN_MAX]; + void (*ctrl_timeout)(struct mt76_dev *dev, int err); bool sg_en; struct mt76u_mcu { @@ -940,6 +941,9 @@ struct mt76_dev { const struct mt76_bus_ops *bus; const struct mt76_driver_ops *drv; const struct mt76_mcu_ops *mcu_ops; + + /* Optional callback to finalize wiphy state before registration. */ + int (*init_wiphy)(struct mt76_dev *dev); struct device *dev; struct device *dma_dev; @@ -2124,6 +2128,9 @@ mt76_vif_link(struct mt76_dev *dev, struct ieee80211_vif *vif, int link_id) if (!link_id) return mlink; + if (link_id >= IEEE80211_MLD_MAX_NUM_LINKS) + return NULL; + return mt76_dereference(mvif->link[link_id], dev); } diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac.h b/drivers/net/wireless/mediatek/mt76/mt76_connac.h index 2aa6078993e9b7..361f29a8b6026e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac.h +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac.h @@ -48,6 +48,11 @@ enum rx_pkt_type { #define MT_TXD_LEN_MSDU_LAST BIT(14) #define MT_TXD_LEN_AMSDU_LAST BIT(15) +/* PCIE part */ +#define PCIE_AER_UNC_STATUS_OFFSET 0x204 +#define PCIE_AER_UNC_MASK_OFFSET 0x208 +#define PCIE_AER_CO_STATUS_OFFSET 0x210 + enum { CMD_CBW_20MHZ = IEEE80211_STA_RX_BW_20, CMD_CBW_40MHZ = IEEE80211_STA_RX_BW_40, @@ -245,7 +250,8 @@ static inline bool is_mt798x(struct mt76_dev *dev) static inline bool is_mt7996(struct mt76_dev *dev) { - return mt76_chip(dev) == 0x7990; + u16 chip = mt76_chip(dev); + return chip == 0x7990 || chip == 0x7991; } static inline bool is_mt7992(struct mt76_dev *dev) diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c index fc9f782032ef2b..976043521f44ff 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c @@ -1069,6 +1069,10 @@ int mt76_connac2_mac_fill_rx_rate(struct mt76_dev *dev, bw = FIELD_GET(MT_CRXV_FRAME_MODE, v2); } + /* the hardware reports NSTS; report the data NSS for STBC frames */ + if (stbc && nss > 1) + nss >>= 1; + switch (*mode) { case MT_PHY_TYPE_CCK: cck = true; @@ -1114,7 +1118,7 @@ int mt76_connac2_mac_fill_rx_rate(struct mt76_dev *dev, case IEEE80211_STA_RX_BW_20: break; case IEEE80211_STA_RX_BW_40: - if (*mode & MT_PHY_TYPE_HE_EXT_SU && + if (*mode == MT_PHY_TYPE_HE_EXT_SU && (idx & MT_PRXV_TX_ER_SU_106T)) { status->bw = RATE_INFO_BW_HE_RU; status->he_ru = diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c index 58b0b15e4fd6ff..b4d81ad02ea92d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c @@ -422,6 +422,10 @@ void mt76_connac_mcu_sta_basic_tlv(struct mt76_dev *dev, struct sk_buff *skb, basic->conn_type = cpu_to_le32(CONNECTION_IBSS_ADHOC); basic->aid = cpu_to_le16(link_sta->sta->aid); break; + case NL80211_IFTYPE_NAN: + case NL80211_IFTYPE_NAN_DATA: + basic->conn_type = cpu_to_le32(CONNECTION_NAN); + break; default: WARN_ON(1); break; @@ -753,7 +757,7 @@ mt76_connac_mcu_sta_he_tlv(struct sk_buff *skb, struct ieee80211_sta *sta) HE_PHY(CAP3_DCM_MAX_CONST_RX_MASK, elem->phy_cap_info[3]); he->dcm_rx_max_nss = HE_PHY(CAP3_DCM_MAX_RX_NSS_2, elem->phy_cap_info[3]); - he->dcm_rx_max_nss = + he->dcm_max_ru = HE_PHY(CAP8_DCM_MAX_RU_MASK, elem->phy_cap_info[8]); he->pkt_ext = 2; @@ -1217,6 +1221,11 @@ int mt76_connac_mcu_uni_add_dev(struct mt76_phy *phy, case NL80211_IFTYPE_ADHOC: basic_req.basic.conn_type = cpu_to_le32(CONNECTION_IBSS_ADHOC); break; + case NL80211_IFTYPE_NAN: + case NL80211_IFTYPE_NAN_DATA: + basic_req.basic.conn_type = cpu_to_le32(CONNECTION_NAN); + basic_req.basic.conn_state = !enable; + break; default: WARN_ON(1); break; @@ -1627,6 +1636,11 @@ int mt76_connac_mcu_uni_add_bss(struct mt76_phy *phy, case NL80211_IFTYPE_ADHOC: basic_req.basic.conn_type = cpu_to_le32(CONNECTION_IBSS_ADHOC); break; + case NL80211_IFTYPE_NAN: + case NL80211_IFTYPE_NAN_DATA: + basic_req.basic.conn_type = cpu_to_le32(CONNECTION_NAN); + basic_req.basic.active = enable; + break; default: WARN_ON(1); break; diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h index 78f633ad81a076..a9a4a87ae0a76b 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h @@ -876,6 +876,7 @@ enum { #define NETWORK_P2P BIT(17) #define NETWORK_IBSS BIT(18) #define NETWORK_WDS BIT(21) +#define NETWORK_NAN BIT(22) #define SCAN_FUNC_RANDOM_MAC BIT(0) #define SCAN_FUNC_RNR_SCAN BIT(3) @@ -888,6 +889,7 @@ enum { #define CONNECTION_IBSS_ADHOC (STA_TYPE_ADHOC | NETWORK_IBSS) #define CONNECTION_WDS (STA_TYPE_WDS | NETWORK_WDS) #define CONNECTION_INFRA_BC (STA_TYPE_BC | NETWORK_INFRA) +#define CONNECTION_NAN (NETWORK_NAN) #define CONN_STATE_DISCONNECT 0 #define CONN_STATE_CONNECT 1 @@ -1074,6 +1076,7 @@ enum { MCU_UNI_EVENT_THERMAL = 0x35, MCU_UNI_EVENT_RSSI_MONITOR = 0x41, MCU_UNI_EVENT_NIC_CAPAB = 0x43, + MCU_UNI_EVENT_NAN = 0x56, MCU_UNI_EVENT_WED_RRO = 0x57, MCU_UNI_EVENT_PER_STA_INFO = 0x6d, MCU_UNI_EVENT_ALL_STA_INFO = 0x6e, @@ -1313,6 +1316,7 @@ enum { MCU_UNI_CMD_FIXED_RATE_TABLE = 0x40, MCU_UNI_CMD_RSSI_MONITOR = 0x41, MCU_UNI_CMD_TESTMODE_CTRL = 0x46, + MCU_UNI_CMD_NAN = 0x56, MCU_UNI_CMD_RRO = 0x57, MCU_UNI_CMD_OFFCH_SCAN_CTRL = 0x58, MCU_UNI_CMD_PER_STA_INFO = 0x6d, diff --git a/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c b/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c index 14ee5b3b94d31e..aa525adb6743c5 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c @@ -848,7 +848,7 @@ int mt76x02_mac_process_rx(struct mt76x02_dev *dev, struct sk_buff *skb, } } - if (WARN_ON_ONCE(len > skb->len)) + if (len > skb->len) return -EINVAL; if (pskb_trim(skb, len)) diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c b/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c index eb92cbf1a284b9..fe7b29ebc0bfb7 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c @@ -257,7 +257,7 @@ void mt7915_eeprom_parse_hw_cap(struct mt7915_dev *dev, nss = min_t(u8, min_t(u8, nss_max, nss), path); mphy->chainmask = BIT(path) - 1; - if (band) + if (band && dev->dbdc_support) mphy->chainmask <<= dev->chainshift; mphy->antenna_mask = BIT(nss) - 1; dev->chainmask |= mphy->chainmask; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/init.c b/drivers/net/wireless/mediatek/mt76/mt7915/init.c index 250c2d2479b0c6..2ab38f9e525810 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/init.c @@ -1272,14 +1272,19 @@ int mt7915_register_device(struct mt7915_dev *dev) ret = mt7915_init_debugfs(&dev->phy); if (ret) - goto unreg_thermal; + goto unreg_ext_phy; ret = mt7915_coredump_register(dev); if (ret) - goto unreg_thermal; + goto unreg_ext_phy; return 0; +unreg_ext_phy: + if (phy2) { + mt7915_unregister_ext_phy(dev); + phy2 = NULL; + } unreg_thermal: mt7915_unregister_thermal(&dev->phy); unreg_dev: @@ -1295,6 +1300,8 @@ free_phy2: void mt7915_unregister_device(struct mt7915_dev *dev) { cancel_work_sync(&dev->dump_work); + cancel_work_sync(&dev->reset_work); + cancel_work_sync(&dev->rc_work); mt7915_unregister_ext_phy(dev); mt7915_coredump_unregister(dev); mt7915_unregister_thermal(&dev->phy); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c index 334c19ab2b22ca..0ea0261f2a29d0 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c @@ -221,7 +221,7 @@ static void mt7915_mac_sta_poll(struct mt7915_dev *dev) rssi[0] = to_rssi(GENMASK(7, 0), val); rssi[1] = to_rssi(GENMASK(15, 8), val); rssi[2] = to_rssi(GENMASK(23, 16), val); - rssi[3] = to_rssi(GENMASK(31, 14), val); + rssi[3] = to_rssi(GENMASK(31, 24), val); msta->ack_signal = mt76_rx_signal(msta->vif->phy->mt76->antenna_mask, rssi); @@ -437,7 +437,7 @@ mt7915_mac_fill_rx(struct mt7915_dev *dev, struct sk_buff *skb, if (v0 & MT_PRXV_HT_AD_CODE) status->enc_flags |= RX_ENC_FLAG_LDPC; - status->chains = mphy->antenna_mask; + status->chains = mt7915_band_chainmask(phy); status->chain_signal[0] = to_rssi(MT_PRXV_RCPI0, v1); status->chain_signal[1] = to_rssi(MT_PRXV_RCPI1, v1); status->chain_signal[2] = to_rssi(MT_PRXV_RCPI2, v1); @@ -2345,8 +2345,10 @@ void mt7915_mac_add_twt_setup(struct ieee80211_hw *hw, } flow->tsf = le64_to_cpu(twt_agrt->twt); - if (mt7915_mcu_twt_agrt_update(dev, msta->vif, flow, MCU_TWT_AGRT_ADD)) + if (mt7915_mcu_twt_agrt_update(dev, msta->vif, flow, MCU_TWT_AGRT_ADD)) { + list_del(&flow->list); goto unlock; + } setup_cmd = TWT_SETUP_CMD_ACCEPT; dev->twt.table_mask |= BIT(table_id); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/main.c b/drivers/net/wireless/mediatek/mt76/mt7915/main.c index 51643a48ed1515..4783e5f52d2293 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/main.c @@ -249,7 +249,7 @@ static int mt7915_add_interface(struct ieee80211_hw *hw, idx = mt76_wcid_alloc(dev->mt76.wcid_mask, mt7915_wtbl_size(dev)); if (idx < 0) { ret = -ENOSPC; - goto out; + goto err; } INIT_LIST_HEAD(&mvif->sta.rc_list); @@ -277,7 +277,17 @@ static int mt7915_add_interface(struct ieee80211_hw *hw, mt7915_mcu_add_sta(dev, vif, NULL, CONN_STATE_PORT_SECURE, true); rcu_assign_pointer(dev->mt76.wcid[idx], &mvif->sta.wcid); + mutex_unlock(&dev->mt76.mutex); + + return 0; + +err: + dev->mt76.vif_mask &= ~BIT_ULL(mvif->mt76.idx); + phy->omac_mask &= ~BIT_ULL(mvif->mt76.omac_idx); + mt7915_mcu_add_dev_info(phy, vif, false); out: + if (phy->monitor_vif == vif) + phy->monitor_vif = NULL; mutex_unlock(&dev->mt76.mutex); return ret; @@ -294,7 +304,6 @@ static void mt7915_remove_interface(struct ieee80211_hw *hw, mt7915_mcu_add_bss_info(phy, vif, false); mt7915_mcu_add_sta(dev, vif, NULL, CONN_STATE_DISCONNECT, false); - mt76_wcid_mask_clear(dev->mt76.wcid_mask, mvif->sta.wcid.idx); mutex_lock(&dev->mt76.mutex); mt76_testmode_reset(phy->mt76, true); @@ -310,6 +319,7 @@ static void mt7915_remove_interface(struct ieee80211_hw *hw, mutex_lock(&dev->mt76.mutex); dev->mt76.vif_mask &= ~BIT_ULL(mvif->mt76.idx); phy->omac_mask &= ~BIT_ULL(mvif->mt76.omac_idx); + mt76_wcid_mask_clear(dev->mt76.wcid_mask, mvif->sta.wcid.idx); mutex_unlock(&dev->mt76.mutex); spin_lock_bh(&dev->mt76.sta_poll_lock); @@ -493,7 +503,7 @@ static int mt7915_config(struct ieee80211_hw *hw, int radio_idx, mt76_rmw_field(dev, MT_DMA_DCR0(band), MT_DMA_DCR0_RXD_G5_EN, enabled); - mt76_rmw_field(dev, MT_DMA_DCR0(band), MT_MDP_DCR0_RX_HDR_TRANS_EN, + mt76_rmw_field(dev, MT_MDP_DCR0, MT_MDP_DCR0_RX_HDR_TRANS_EN, !dev->monitor_mask); mt76_testmode_reset(phy->mt76, true); mt76_wr(dev, MT_WF_RFCR(band), rxfilter); @@ -1128,7 +1138,7 @@ mt7915_set_antenna(struct ieee80211_hw *hw, int radio_idx, u32 tx_ant, u32 rx_an struct mt7915_dev *dev = mt7915_hw_dev(hw); struct mt7915_phy *phy = mt7915_hw_phy(hw); int max_nss = hweight8(hw->wiphy->available_antennas_tx); - u8 chainshift = dev->chainshift; + u8 shift = mt7915_band_chainshift(phy); u8 band = phy->mt76->band_idx; if (!tx_ant || tx_ant != rx_ant || ffs(tx_ant) > max_nss) @@ -1141,9 +1151,9 @@ mt7915_set_antenna(struct ieee80211_hw *hw, int radio_idx, u32 tx_ant, u32 rx_an /* handle a variant of mt7916/mt7981 which has 3T3R but nss2 on 5 GHz band */ if ((is_mt7916(&dev->mt76) || is_mt7981(&dev->mt76)) && band && hweight8(tx_ant) == max_nss) - phy->mt76->chainmask = (dev->chainmask >> chainshift) << chainshift; + phy->mt76->chainmask = (dev->chainmask >> shift) << shift; else - phy->mt76->chainmask = tx_ant << (chainshift * band); + phy->mt76->chainmask = tx_ant << shift; mt76_set_stream_caps(phy->mt76, true); mt7915_set_stream_vht_txbf_caps(phy); @@ -1743,7 +1753,7 @@ mt7915_net_fill_forward_path(struct ieee80211_hw *hw, path->mtk_wdma.wdma_idx = wed->wdma_idx; path->mtk_wdma.bss = mvif->mt76.idx; path->mtk_wdma.wcid = is_mt7915(&dev->mt76) ? msta->wcid.idx : 0x3ff; - path->mtk_wdma.queue = phy != &dev->phy; + path->mtk_wdma.queue = phy->mt76->band_idx; ctx->dev = NULL; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c index bbb2fedacb25af..88955aed62e2bd 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c @@ -51,7 +51,7 @@ mt7915_mcu_get_sta_nss(u16 mcs_map) break; } - return nss - 1; + return nss ? nss - 1 : 0; } static void @@ -576,9 +576,9 @@ mt7915_mcu_bss_ra_tlv(struct sk_buff *skb, struct ieee80211_vif *vif, ra->rx_streams = max_nss; ra->algo = 4; ra->train_up_rule = 2; - ra->train_up_high_thres = 110; - ra->train_up_rule_rssi = -70; - ra->low_traffic_thres = 2; + ra->train_up_high_thres = cpu_to_le16(110); + ra->train_up_rule_rssi = cpu_to_le16(-70); + ra->low_traffic_thres = cpu_to_le16(2); ra->phy_cap = cpu_to_le32(0xfdf); ra->interval = cpu_to_le32(500); ra->fast_interval = cpu_to_le32(100); @@ -902,7 +902,7 @@ mt7915_mcu_sta_he_tlv(struct sk_buff *skb, struct ieee80211_sta *sta, HE_PHY(CAP3_DCM_MAX_CONST_RX_MASK, elem->phy_cap_info[3]); he->dcm_rx_max_nss = HE_PHY(CAP3_DCM_MAX_RX_NSS_2, elem->phy_cap_info[3]); - he->dcm_rx_max_nss = + he->dcm_max_ru = HE_PHY(CAP8_DCM_MAX_RU_MASK, elem->phy_cap_info[8]); he->pkt_ext = 2; @@ -1625,7 +1625,7 @@ mt7915_mcu_sta_rate_ctrl_tlv(struct sk_buff *skb, struct mt7915_dev *dev, cap |= STA_CAP_VHT_SGI_160; if (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_TXSTBC) cap |= STA_CAP_VHT_TX_STBC; - if (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_RXSTBC_1) + if (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_RXSTBC_MASK) cap |= STA_CAP_VHT_RX_STBC; if (mvif->cap.vht_ldpc && (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_RXLDPC)) @@ -2804,7 +2804,7 @@ int mt7915_mcu_set_chan_info(struct mt7915_phy *phy, int cmd) .center_ch = ieee80211_frequency_to_channel(freq1), .bw = mt76_connac_chan_bw(chandef), .tx_path_num = hweight16(phy->mt76->chainmask), - .rx_path = phy->mt76->chainmask >> (dev->chainshift * band), + .rx_path = mt7915_band_chainmask(phy), .band_idx = band, .channel_band = ch_band[chandef->chan->band], }; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h index 22f73a5ed42590..7c472062a90e1c 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h @@ -318,9 +318,9 @@ struct bss_info_ra { u8 antenna_idx; u8 train_up_rule; u8 rsv[3]; - unsigned short train_up_high_thres; - short train_up_rule_rssi; - unsigned short low_traffic_thres; + __le16 train_up_high_thres; + __le16 train_up_rule_rssi; + __le16 low_traffic_thres; __le16 max_phyrate; __le32 phy_cap; __le32 interval; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h index bf1d915a3ca238..43479f6487ede5 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h @@ -397,6 +397,24 @@ mt7915_ext_phy(struct mt7915_dev *dev) return phy->priv; } +/* without dbdc, the chainmask is stored unshifted, even if the phy is + * bound to band 1 + */ +static inline u8 mt7915_band_chainshift(struct mt7915_phy *phy) +{ + struct mt7915_dev *dev = phy->dev; + + if (!dev->dbdc_support) + return 0; + + return phy->mt76->band_idx * dev->chainshift; +} + +static inline u16 mt7915_band_chainmask(struct mt7915_phy *phy) +{ + return phy->mt76->chainmask >> mt7915_band_chainshift(phy); +} + static inline u32 mt7915_check_adie(struct mt7915_dev *dev, bool sku) { u32 mask = sku ? MT_CONNINFRA_SKU_MASK : MT_ADIE_TYPE_MASK; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/pci.c b/drivers/net/wireless/mediatek/mt76/mt7915/pci.c index f6b03211a879b3..8007e620048b33 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/pci.c @@ -135,7 +135,6 @@ static int mt7915_pci_probe(struct pci_dev *pdev, mdev = &dev->mt76; mt7915_wfsys_reset(dev); - hif2 = mt7915_pci_init_hif2(pdev); ret = mt7915_mmio_wed_init(dev, pdev, true, &irq); if (ret < 0) @@ -145,16 +144,20 @@ static int mt7915_pci_probe(struct pci_dev *pdev, hif2 = mt7915_pci_init_hif2(pdev); ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); - if (ret < 0) + if (ret < 0) { + mt7915_put_hif2(hif2); goto free_device; + } irq = pdev->irq; } ret = devm_request_irq(mdev->dev, irq, mt7915_irq_handler, IRQF_SHARED, KBUILD_MODNAME, dev); - if (ret) + if (ret) { + mt7915_put_hif2(hif2); goto free_wed_or_irq_vector; + } /* master switch of PCIe tnterrupt enable */ mt76_wr(dev, MT_PCIE_MAC_INT_ENABLE, 0xff); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/soc.c b/drivers/net/wireless/mediatek/mt76/mt7915/soc.c index 54ff6de96f3ea9..13fba2a061c78e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/soc.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/soc.c @@ -908,7 +908,7 @@ static void mt7986_wmac_clock_enable(struct mt7915_dev *dev, u32 adie_type) read_poll_timeout(mt76_rr, cur, !(cur & MT_SLP_CTRL_BSY_MASK), USEC_PER_MSEC, 50 * USEC_PER_MSEC, false, - dev, MT_ADIE_SLP_CTRL_CK0(0)); + dev, MT_ADIE_SLP_CTRL_CK0(1)); } mt76_wmac_spi_unlock(dev); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/testmode.c b/drivers/net/wireless/mediatek/mt76/mt7915/testmode.c index 618a5c2bdd29fa..7576973f4d4ef3 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/testmode.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/testmode.c @@ -694,9 +694,7 @@ mt7915_tm_set_params(struct mt76_phy *mphy, struct nlattr **tb, { struct mt76_testmode_data *td = &mphy->test; struct mt7915_phy *phy = mphy->priv; - struct mt7915_dev *dev = phy->dev; - u32 chainmask = mphy->chainmask, changed = 0; - bool ext_phy = phy != &dev->phy; + u32 chainmask = mt7915_band_chainmask(phy), changed = 0; int i; BUILD_BUG_ON(NUM_TM_CHANGED >= 32); @@ -705,7 +703,6 @@ mt7915_tm_set_params(struct mt76_phy *mphy, struct nlattr **tb, td->state == MT76_TM_STATE_OFF) return 0; - chainmask = ext_phy ? chainmask >> dev->chainshift : chainmask; if (td->tx_antenna_mask > chainmask) return -EINVAL; diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c index f7d54472da1b10..e69978184f68c6 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c @@ -156,7 +156,7 @@ static void mt7921_mac_sta_poll(struct mt792x_dev *dev) rssi[0] = to_rssi(GENMASK(7, 0), val); rssi[1] = to_rssi(GENMASK(15, 8), val); rssi[2] = to_rssi(GENMASK(23, 16), val); - rssi[3] = to_rssi(GENMASK(31, 14), val); + rssi[3] = to_rssi(GENMASK(31, 24), val); mlink->ack_signal = mt76_rx_signal(msta->vif->phy->mt76->antenna_mask, rssi); @@ -674,6 +674,9 @@ void mt7921_mac_reset_work(struct work_struct *work) cancel_work_sync(&pm->wake_work); for (i = 0; i < 10; i++) { + if (atomic_read(&dev->mt76.bus_hung)) + return; + mutex_lock(&dev->mt76.mutex); ret = mt792x_dev_reset(dev); mutex_unlock(&dev->mt76.mutex); diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c index 7728c5ae679145..4617178fb1c464 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c @@ -600,6 +600,108 @@ failed: return err; } +static u32 mt7921_aer_rr(struct mt76_dev *mdev, u32 offset) +{ + return 0; +} + +static void mt7921_aer_wr(struct mt76_dev *mdev, u32 offset, u32 val) +{ + ; +} + +static u32 mt791_aer_rmw(struct mt76_dev *mdev, u32 offset, u32 mask, u32 val) +{ + return 0; +} + +static const struct mt76_bus_ops mt7921_aer_bus_hung_ops = { + .rr = mt7921_aer_rr, + .wr = mt7921_aer_wr, + .rmw = mt791_aer_rmw, + .type = MT76_BUS_MMIO +}; + +static void mt7921_pci_set_aer_bus_hung_ops(struct mt792x_dev *dev) +{ + if (READ_ONCE(dev->mt76.bus) == &mt7921_aer_bus_hung_ops) + return; + + atomic_set(&dev->mt76.bus_hung, true); + WRITE_ONCE(dev->mt76.bus, &mt7921_aer_bus_hung_ops); +} + +static pci_ers_result_t mt7921_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct mt76_dev *mdev = pci_get_drvdata(pdev); + struct mt792x_dev *dev = container_of(mdev, struct mt792x_dev, mt76); + u32 aer_unc_val = 0, aer_co_val = 0; + + dev_err(mdev->dev, "PCIE error detect state: %d\n", state); + + /* Clear SW IRQ tasklet first */ + tasklet_kill(&mdev->irq_tasklet); + + if (state == pci_channel_io_perm_failure) { + mt7921_pci_set_aer_bus_hung_ops(dev); + return PCI_ERS_RESULT_DISCONNECT; + } + + pci_read_config_dword(pdev, PCIE_AER_UNC_STATUS_OFFSET, &aer_unc_val); + pci_read_config_dword(pdev, PCIE_AER_CO_STATUS_OFFSET, &aer_co_val); + + dev_warn(mdev->dev, "PCIE_AER_UNC_STATUS_OFFSET: 0x%x\n", aer_unc_val); + dev_warn(mdev->dev, "PCIE_AER_CO_STATUS_OFFSET: 0x%x\n", aer_co_val); + + /** + * Due to this error is from link error and this AER is un-correctable, + * so can't covered by device + **/ + if (aer_unc_val != 0) { + mt7921_pci_set_aer_bus_hung_ops(dev); + return PCI_ERS_RESULT_DISCONNECT; + } + + /** + * Try to recover it when state is pci_channel_io_frozen or + * AER is correctable error + **/ + if (state == pci_channel_io_frozen || aer_co_val != 0) { + /* Disable PCIE activity first. */ + pci_disable_device(pdev); + return PCI_ERS_RESULT_NEED_RESET; + } + + return PCI_ERS_RESULT_NONE; +} + +static pci_ers_result_t mt7921_slot_reset(struct pci_dev *pdev) +{ + struct mt76_dev *mdev = pci_get_drvdata(pdev); + int ret = 0; + + ret = pci_enable_device_mem(pdev); + + if (ret) { + dev_err(mdev->dev, "pci_enable_device_mem failed: %d\n", ret); + return PCI_ERS_RESULT_DISCONNECT; + } + + pci_set_master(pdev); + pci_restore_state(pdev); + pci_save_state(pdev); + /* Also try do the vendor reset to let it more clear. */ + mt792x_reset(mdev); + + return PCI_ERS_RESULT_RECOVERED; +} + +static const struct pci_error_handlers mt7921_err_handler = { + .error_detected = mt7921_error_detected, + .slot_reset = mt7921_slot_reset, +}; + static void mt7921_pci_shutdown(struct pci_dev *pdev) { mt7921_pci_remove(pdev); @@ -614,6 +716,7 @@ static struct pci_driver mt7921_pci_driver = { .remove = mt7921_pci_remove, .shutdown = mt7921_pci_shutdown, .driver.pm = pm_sleep_ptr(&mt7921_pm_ops), + .err_handler = &mt7921_err_handler, }; module_pci_driver(mt7921_pci_driver); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/Makefile b/drivers/net/wireless/mediatek/mt76/mt7925/Makefile index 8f1078ce32316f..f9dcc0bba393ca 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/Makefile +++ b/drivers/net/wireless/mediatek/mt76/mt7925/Makefile @@ -4,7 +4,7 @@ obj-$(CONFIG_MT7925_COMMON) += mt7925-common.o obj-$(CONFIG_MT7925E) += mt7925e.o obj-$(CONFIG_MT7925U) += mt7925u.o -mt7925-common-y := mac.o mcu.o regd.o main.o init.o debugfs.o +mt7925-common-y := mac.o mcu.o regd.o main.o init.o debugfs.o nan.o mt7925-common-$(CONFIG_NL80211_TESTMODE) += testmode.o mt7925e-y := pci.o pci_mac.o pci_mcu.o mt7925u-y := usb.o diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c index 9b58ffec36493d..b52b6783306042 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c @@ -145,7 +145,7 @@ static void mt7925_mac_sta_poll(struct mt792x_dev *dev) rssi[0] = to_rssi(GENMASK(7, 0), val); rssi[1] = to_rssi(GENMASK(15, 8), val); rssi[2] = to_rssi(GENMASK(23, 16), val); - rssi[3] = to_rssi(GENMASK(31, 14), val); + rssi[3] = to_rssi(GENMASK(31, 24), val); mlink->ack_signal = mt76_rx_signal(msta->vif->phy->mt76->antenna_mask, rssi); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/main.c b/drivers/net/wireless/mediatek/mt76/mt7925/main.c index 3beb2c1fbec9c5..9f080da13341b3 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/main.c @@ -187,19 +187,21 @@ mt7925_init_eht_caps(struct mt792x_phy *phy, enum nl80211_band band, eht_cap_elem->phy_cap_info[0] |= IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ; + val = (sts > 3) ? sts - 1 : 3; + eht_cap_elem->phy_cap_info[0] |= - u8_encode_bits(u8_get_bits(sts - 1, BIT(0)), + u8_encode_bits(u8_get_bits(val, BIT(0)), IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK); eht_cap_elem->phy_cap_info[1] = - u8_encode_bits(u8_get_bits(sts - 1, GENMASK(2, 1)), + u8_encode_bits(u8_get_bits(val, GENMASK(2, 1)), IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK) | - u8_encode_bits(sts - 1, + u8_encode_bits(val, IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK); if (band == NL80211_BAND_6GHZ && is_320mhz_supported(&phy->dev->mt76)) eht_cap_elem->phy_cap_info[1] |= - u8_encode_bits(sts - 1, + u8_encode_bits(val, IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_320MHZ_MASK); eht_cap_elem->phy_cap_info[2] = @@ -1828,9 +1830,15 @@ static int mt7925_set_sar_specs(struct ieee80211_hw *hw, int err; mt792x_mutex_acquire(dev); + err = mt7925_mcu_set_clc(dev, dev->mt76.alpha2, + dev->country_ie_env); + if (err < 0) + goto out; + err = mt7925_set_tx_sar_pwr(hw, sar); - mt792x_mutex_release(dev); +out: + mt792x_mutex_release(dev); return err; } @@ -2494,6 +2502,40 @@ static void mt7925_stop(struct ieee80211_hw *hw, bool suspend) mt792x_stop(hw, suspend); } +static void mt7925_sta_pre_rcu_remove(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta) +{ + struct mt76_phy *phy = hw->priv; + struct mt76_dev *dev = phy->dev; + struct mt76_wcid *wcid = (struct mt76_wcid *)sta->drv_priv; + + mutex_lock(&dev->mutex); + spin_lock_bh(&dev->status_lock); + + if (ieee80211_vif_is_mld(vif)) { + struct mt792x_sta *msta = (struct mt792x_sta *)sta->drv_priv; + struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; + unsigned long valid = mvif->valid_links; + struct mt792x_link_sta *mlink; + unsigned int link_id; + + for_each_set_bit(link_id, &valid, IEEE80211_MLD_MAX_NUM_LINKS) { + mlink = mt792x_sta_to_link(msta, link_id); + if (!mlink || !mlink->wcid.sta) + continue; + if (mlink->wcid.idx < ARRAY_SIZE(dev->wcid)) + rcu_assign_pointer(dev->wcid[mlink->wcid.idx], + NULL); + } + } else { + rcu_assign_pointer(dev->wcid[wcid->idx], NULL); + } + + spin_unlock_bh(&dev->status_lock); + mutex_unlock(&dev->mutex); +} + const struct ieee80211_ops mt7925_ops = { .tx = mt792x_tx, .start = mt7925_start, @@ -2506,7 +2548,7 @@ const struct ieee80211_ops mt7925_ops = { .start_ap = mt7925_start_ap, .stop_ap = mt7925_stop_ap, .sta_state = mt76_sta_state, - .sta_pre_rcu_remove = mt76_sta_pre_rcu_remove, + .sta_pre_rcu_remove = mt7925_sta_pre_rcu_remove, .set_key = mt7925_set_key, .sta_set_decap_offload = mt7925_sta_set_decap_offload, #if IS_ENABLED(CONFIG_IPV6) diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c index cb265a6fc7adbd..53c142345a0861 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c @@ -2174,6 +2174,8 @@ int mt7925_get_txpwr_info(struct mt792x_dev *dev, u8 band_idx, struct mt7925_txp int mt7925_mcu_set_sniffer(struct mt792x_dev *dev, struct ieee80211_vif *vif, bool enable) { + struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; + struct ieee80211_chanctx_conf *ctx = mvif->bss_conf.mt76.ctx; struct { struct { u8 band_idx; @@ -2196,6 +2198,15 @@ int mt7925_mcu_set_sniffer(struct mt792x_dev *dev, struct ieee80211_vif *vif, }, }; + if (is_mt7927(&dev->mt76)) { + struct ieee80211_channel *chan; + + chan = ctx ? ctx->def.chan : mvif->phy->mt76->chandef.chan; + + if (chan) + req.hdr.band_idx = mt7927_band_idx(chan->band); + } + return mt76_mcu_send_msg(&dev->mt76, MCU_UNI_CMD(SNIFFER), &req, sizeof(req), true); } @@ -2255,6 +2266,9 @@ int mt7925_mcu_config_sniffer(struct mt792x_vif *vif, }, }; + if (is_mt7927(mphy->dev)) + req.hdr.band_idx = mt7927_band_idx(chandef->chan->band); + if (chandef->chan->band < ARRAY_SIZE(ch_band)) req.tlv.ch_band = ch_band[chandef->chan->band]; if (chandef->width < ARRAY_SIZE(ch_width)) @@ -3829,7 +3843,7 @@ mt7925_mcu_rate_txpower_band(struct mt76_phy *phy, memcpy(tx_power_tlv->alpha2, dev->alpha2, sizeof(dev->alpha2)); tx_power_tlv->n_chan = num_ch; tx_power_tlv->tag = cpu_to_le16(0x1); - tx_power_tlv->len = cpu_to_le16(msg_len); + tx_power_tlv->len = cpu_to_le16(msg_len - 4); switch (band) { case NL80211_BAND_2GHZ: diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/nan.c b/drivers/net/wireless/mediatek/mt76/mt7925/nan.c new file mode 100644 index 00000000000000..74db344a679694 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7925/nan.c @@ -0,0 +1,927 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* Copyright (C) 2025-2026 MediaTek Inc. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mt7925.h" +#include "mcu.h" +#include "nan.h" +#include "regd.h" + +static void mt7925_nan_set_5g_channel(struct mt792x_dev *dev, + struct mt7925_nan_enable_req_tlv *req, + struct cfg80211_nan_conf *conf) +{ + struct ieee80211_channel *chan; + u32 ch5g = 0; + + chan = conf->band_cfgs[NL80211_BAND_5GHZ].chan; + + if (!chan) + return; + + if (!mt7925_regd_is_valid_channel(dev, NL80211_BAND_5GHZ, chan)) + return; + + req->config_5g_channel = 1; + + if (chan->hw_value == NAN_5G_LOW_DISC_CHANNEL) + ch5g |= BIT(0); + else if (chan->hw_value == NAN_5G_HIGH_DISC_CHANNEL) + ch5g |= BIT(1); + + req->channel_5g_val = cpu_to_le32(ch5g); +} + +static void mt7925_nan_set_cluster_id(struct mt7925_nan_enable_req_tlv *req, + const u8 *cluster_id) +{ + if (!cluster_id) + return; + + req->cluster_high = cpu_to_le16(cluster_id[4] | cluster_id[5] << 8); + req->cluster_low = cpu_to_le16((u16)cluster_id[3]); +} + +static void mt7925_nan_set_dw_interval(struct mt7925_nan_enable_req_tlv *req, + struct cfg80211_nan_conf *conf) +{ + if (conf->band_cfgs[NL80211_BAND_2GHZ].awake_dw_interval > 0) { + req->config_dw.config_2dot4g_dw_band = 1; + req->config_dw.dw_2dot4g_interval_val = + cpu_to_le32(conf->band_cfgs[NL80211_BAND_2GHZ].awake_dw_interval); + } + + if (conf->band_cfgs[NL80211_BAND_5GHZ].awake_dw_interval > 0) { + req->config_dw.config_5g_dw_band = 1; + req->config_dw.dw_5g_interval_val = + cpu_to_le32(conf->band_cfgs[NL80211_BAND_5GHZ].awake_dw_interval); + } +} + +static void mt7925_nan_set_disc_beacon(struct mt7925_nan_enable_req_tlv *req, + struct cfg80211_nan_conf *conf) +{ + if (conf->discovery_beacon_interval > 0) { + req->config_2dot4g_beacons = true; + req->beacon_2dot4g_val = conf->discovery_beacon_interval; + } +} + +static void mt7925_nan_set_rssi_thresholds(struct mt7925_nan_enable_req_tlv *req, + struct cfg80211_nan_conf *conf) +{ + if (conf->band_cfgs[NL80211_BAND_2GHZ].chan) { + req->config_2dot4g_rssi_close = 1; + req->rssi_close_2dot4g_val = + abs(conf->band_cfgs[NL80211_BAND_2GHZ].rssi_close); + req->config_2dot4g_rssi_middle = 1; + req->rssi_middle_2dot4g_val = + abs(conf->band_cfgs[NL80211_BAND_2GHZ].rssi_middle); + } + + if (conf->band_cfgs[NL80211_BAND_5GHZ].chan) { + req->config_5g_rssi_close = 1; + req->rssi_close_5g_val = + abs(conf->band_cfgs[NL80211_BAND_5GHZ].rssi_close); + req->config_5g_rssi_middle = 1; + req->rssi_middle_5g_val = + abs(conf->band_cfgs[NL80211_BAND_5GHZ].rssi_middle); + } +} + +static void mt7925_nan_set_scan_params(struct mt7925_nan_enable_req_tlv *req, + struct cfg80211_nan_conf *conf) +{ + req->scan_params_val.scan_period[0] = + cpu_to_le16(conf->scan_period < 255 ? conf->scan_period : 255); + req->scan_params_val.dwell_time[0] = + conf->scan_dwell_time < 255 ? conf->scan_dwell_time : 255; +} + +static u16 +mt7925_nan_avail_attr_ctrl(const struct ieee80211_nan_sched_cfg *sched) +{ + if (sched->avail_blob_len < NAN_AVAIL_ATTR_CTRL_OFFSET + 2) + return 0; + + return sched->avail_blob[NAN_AVAIL_ATTR_CTRL_OFFSET] | + sched->avail_blob[NAN_AVAIL_ATTR_CTRL_OFFSET + 1] << 8; +} + +static void +mt7925_nan_update_conf(struct mt792x_vif *mvif, + const struct cfg80211_nan_conf *conf) +{ + mvif->nan.conf.master_pref = conf->master_pref; + mvif->nan.conf.bands = conf->bands; + mvif->nan.conf.discovery_beacon_interval = + conf->discovery_beacon_interval; + mvif->nan.conf.enable_dw_notification = + conf->enable_dw_notification; + + memcpy(mvif->nan.conf.cluster_id, conf->cluster_id, ETH_ALEN); +} + +int mt7925_nan_enable(struct ieee80211_vif *vif, + struct mt792x_dev *dev, + struct cfg80211_nan_conf *conf) +{ + struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; + struct mt76_dev *mdev = &dev->mt76; + struct { + u8 rsv[4]; + struct mt7925_nan_enable_req_tlv nan_req_tlv; + } nan_cmd = { + .rsv = { 0 }, + .nan_req_tlv = { + .tag = cpu_to_le16(NAN_UNI_CMD_ENABLE_REQUEST), + .len = cpu_to_le16(sizeof(struct mt7925_nan_enable_req_tlv)), + .config_random_factor_force = 0, + .random_factor_force_val = 0, + .config_hop_count_force = 0, + .hop_count_force_val = 0, + }, + }; + struct mt7925_nan_enable_req_tlv *p_nan_req_tlv = &nan_cmd.nan_req_tlv; + + if (!vif || !dev || !conf) + return -EINVAL; + + p_nan_req_tlv->master_pref = conf->master_pref; + + mt7925_nan_set_5g_channel(dev, p_nan_req_tlv, conf); + mt7925_nan_set_cluster_id(p_nan_req_tlv, conf->cluster_id); + mt7925_nan_set_dw_interval(p_nan_req_tlv, conf); + mt7925_nan_set_disc_beacon(p_nan_req_tlv, conf); + mt7925_nan_set_rssi_thresholds(p_nan_req_tlv, conf); + mt7925_nan_set_scan_params(p_nan_req_tlv, conf); + + mt7925_nan_update_conf(mvif, conf); + + return mt76_mcu_send_msg(mdev, MCU_UNI_CMD(NAN), &nan_cmd, sizeof(nan_cmd), true); +} + +int mt7925_nan_disable(struct ieee80211_vif *vif, struct mt792x_dev *dev) +{ + struct mt76_dev *mdev = &dev->mt76; + struct { + u8 rsv[4]; + struct tlv nan_dis_tlv; + } nan_cmd = { + .rsv = { 0 }, + .nan_dis_tlv = { + .tag = cpu_to_le16(NAN_UNI_CMD_DISABLE_REQUEST), + .len = cpu_to_le16(sizeof(struct tlv)), + }, + }; + + if (!dev) + return -EINVAL; + + return mt76_mcu_send_msg(mdev, MCU_UNI_CMD(NAN), &nan_cmd, sizeof(nan_cmd), true); +} + +static int +mt7925_nan_mp_tlv(struct sk_buff *skb, u8 master_pref) +{ + struct mt7925_nan_master_preference_tlv *mp_tlv = NULL; + struct tlv *tlv = NULL; + + if (!skb) + return -EINVAL; + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_SET_MASTER_PREFERENCE, + sizeof(struct mt7925_nan_master_preference_tlv)); + if (!tlv) + return -ENOMEM; + + mp_tlv = (struct mt7925_nan_master_preference_tlv *)tlv; + + if (master_pref > NAN_MAX_MASTER_PREFERENCE) + return 0; + + mp_tlv->master_preference = master_pref; + + return 0; +} + +static int +mt7925_nan_dw_tlv(struct sk_buff *skb, struct cfg80211_nan_conf *conf) +{ + struct mt7925_nan_dw_interval_tlv *dw_tlv = NULL; + struct tlv *tlv = NULL; + u16 interval; + + if (!skb || !conf) + return -EINVAL; + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_SET_DW_INTERVAL, + sizeof(struct mt7925_nan_dw_interval_tlv)); + + if (!tlv) + return -ENOMEM; + + dw_tlv = (struct mt7925_nan_dw_interval_tlv *)tlv; + + /* Set DW interval for 2.4GHz and 5GHz bands if available */ + if (conf->band_cfgs[NL80211_BAND_2GHZ].awake_dw_interval > 0) { + dw_tlv->dw_interval = conf->band_cfgs[NL80211_BAND_2GHZ].awake_dw_interval; + } else if (conf->band_cfgs[NL80211_BAND_5GHZ].awake_dw_interval > 0) { + dw_tlv->dw_interval = conf->band_cfgs[NL80211_BAND_5GHZ].awake_dw_interval; + } else { + /* Fallback to a default value or log a warning */ + dw_tlv->dw_interval = NAN_DEFAULT_DW_INTERVAL; + } + + /* Validate and set NAN Discovery Beacon Interval */ + interval = conf->discovery_beacon_interval > 0 ? + conf->discovery_beacon_interval : + NAN_DEFAULT_DISC_BCN_INTERVAL; + + dw_tlv->disc_bcn_interval = cpu_to_le16(interval); + + return 0; +} + +static int +mt7925_nan_cluster_id_tlv(struct sk_buff *skb, const u8 *cluster_id) +{ + struct mt7925_nan_cluster_id_tlv *cluster_tlv = NULL; + struct tlv *tlv = NULL; + + if (!skb || !cluster_id) + return -EINVAL; + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_SET_CLUSTER_ID, + sizeof(struct mt7925_nan_cluster_id_tlv)); + + if (!tlv) + return -ENOMEM; + + cluster_tlv = (struct mt7925_nan_cluster_id_tlv *)tlv; + + memcpy(cluster_tlv->cluster_id, cluster_id, ETH_ALEN); + + return 0; +} + +static int +mt7925_nan_sync_rssi_tlv(struct sk_buff *skb, struct cfg80211_nan_conf *conf) +{ + struct mt7925_nan_sync_rssi_tlv *rssi_tlv = NULL; + struct tlv *tlv = NULL; + + if (!skb || !conf) + return -EINVAL; + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_SET_SYNC_RSSI, + sizeof(struct mt7925_nan_sync_rssi_tlv)); + + if (!tlv) + return -ENOMEM; + + rssi_tlv = (struct mt7925_nan_sync_rssi_tlv *)tlv; + + if (conf->band_cfgs[NL80211_BAND_2GHZ].chan) { + rssi_tlv->rssi_close_2g = + conf->band_cfgs[NL80211_BAND_2GHZ].rssi_close; + rssi_tlv->rssi_middle_2g = + conf->band_cfgs[NL80211_BAND_2GHZ].rssi_middle; + } + + if (conf->band_cfgs[NL80211_BAND_5GHZ].chan) { + rssi_tlv->rssi_close_5g = + conf->band_cfgs[NL80211_BAND_5GHZ].rssi_close; + rssi_tlv->rssi_middle_5g = + conf->band_cfgs[NL80211_BAND_5GHZ].rssi_middle; + } + + return 0; +} + +int mt7925_nan_change_configure(struct ieee80211_vif *vif, + struct mt792x_dev *dev, + struct cfg80211_nan_conf *conf) +{ + struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; + struct mt7925_nan_common_hdr *hdr = NULL; + struct mt76_dev *mdev = &dev->mt76; + struct sk_buff *skb = NULL; + + if (!vif || !dev || !conf) + return -EINVAL; + + skb = mt76_mcu_msg_alloc(mdev, NULL, MT7925_NAN_CONF_MAX_SIZE); + if (!skb) + return -ENOMEM; + + hdr = (struct mt7925_nan_common_hdr *)skb_put(skb, sizeof(*hdr)); + memset(hdr, 0, sizeof(*hdr)); + + if (mt7925_nan_mp_tlv(skb, conf->master_pref) || + mt7925_nan_dw_tlv(skb, conf) || + mt7925_nan_cluster_id_tlv(skb, conf->cluster_id) || + mt7925_nan_sync_rssi_tlv(skb, conf)) { + dev_kfree_skb(skb); + return -ENOMEM; + } + + mt7925_nan_update_conf(mvif, conf); + + return mt76_mcu_skb_send_msg(mdev, skb, + MCU_UNI_CMD(NAN), true); +} + +static void +mt7925_nan_handle_dw_ind(struct mt792x_dev *dev, struct tlv *tlv) +{ + struct ieee80211_channel *chan; + struct nan_rpt_dw_evt *evt; + struct wireless_dev *wdev; + u16 len, channel, dw_num; + struct mt792x_vif *mvif; + enum nl80211_band band; + int freq; + + if (!dev || !tlv) + return; + + len = le16_to_cpu(tlv->len); + if (len < sizeof(*tlv) + sizeof(*evt)) { + dev_warn(dev->mt76.dev, + "nan: short dw event tlv len=%u\n", len); + return; + } + + if (!dev->nan_vif || !ieee80211_vif_nan_started(dev->nan_vif)) + return; + + wdev = ieee80211_vif_to_wdev(dev->nan_vif); + if (!wdev) + return; + + mvif = (struct mt792x_vif *)dev->nan_vif->drv_priv; + if (!mvif->nan.conf.enable_dw_notification) + return; + + evt = (struct nan_rpt_dw_evt *)tlv->data; + channel = le16_to_cpu(evt->channel); + dw_num = le16_to_cpu(evt->dw_num); + + band = channel > 13 ? NL80211_BAND_5GHZ : NL80211_BAND_2GHZ; + freq = ieee80211_channel_to_frequency(channel, band); + chan = ieee80211_get_channel(dev->mt76.hw->wiphy, freq); + if (!chan) { + dev_dbg(dev->mt76.dev, + "nan: no channel for dw end event ch=%u dw=%u\n", + channel, dw_num); + return; + } + + cfg80211_next_nan_dw_notif(wdev, chan, GFP_KERNEL); +} + +static void +mt7925_nan_mcu_handle_de_event(struct mt792x_dev *dev, struct tlv *tlv) +{ + u8 cluster_id[ETH_ALEN] __aligned(2) = {0x50, 0x6f, 0x9a, 0x01, 0x00, 0x00}; + struct mt7925_nan_de_event *de_evt = NULL; + u16 len; + + if (!dev || !tlv) { + if (dev) + dev_warn(dev->mt76.dev, "nan: failed to parse TLV\n"); + return; + } + + len = le16_to_cpu(tlv->len); + if (len < sizeof(*tlv) + sizeof(*de_evt)) { + dev_warn(dev->mt76.dev, + "nan: short de_event tlv len=%u\n", len); + return; + } + + de_evt = (struct mt7925_nan_de_event *)tlv->data; + if (!de_evt) { + dev_warn(dev->mt76.dev, "nan: missing DE event payload\n"); + return; + } + + if (de_evt->event_type == NAN_EVENT_ID_DISC_MAC_ADDR) + return; + + memcpy(cluster_id, de_evt->cluster_id, ETH_ALEN); + + dev_dbg(dev->mt76.dev, "nan: evt=%u cluster=%pM\n", + de_evt->event_type, de_evt->cluster_id); + + if (de_evt->event_type != NAN_EVENT_ID_JOINED_CLUSTER) + return; + + if (!ieee80211_vif_nan_started(dev->nan_vif)) { + dev_warn(dev->mt76.dev, "nan: joined-cluster event but NAN not started\n"); + return; + } + + dev_dbg(dev->mt76.dev, "nan: anchor_master_rank=%*phN\n", + NAN_ANCHOR_MASTER_RANK_NUM, de_evt->anchor_master_rank); + + dev_dbg(dev->mt76.dev, "nan: own_nmi=%pM master_nmi=%pM\n", + de_evt->own_nmi, de_evt->master_nmi); + + ieee80211_nan_cluster_joined(dev->nan_vif, cluster_id, true, GFP_KERNEL); +} + +void mt7925_nan_mcu_event(struct mt792x_dev *dev, struct sk_buff *skb) +{ + struct tlv *tlv; + u32 tlv_len; + + if (!dev || !skb) + return; + + if (skb->len < sizeof(struct mt7925_mcu_rxd) + 4) + return; + + skb_pull(skb, sizeof(struct mt7925_mcu_rxd) + 4); + tlv = (struct tlv *)skb->data; + tlv_len = skb->len; + + while (tlv_len >= sizeof(*tlv)) { + u16 len = le16_to_cpu(tlv->len); + + if (len < sizeof(*tlv) || len > tlv_len) + break; + + switch (le16_to_cpu(tlv->tag)) { + case NAN_UNI_EVENT_ID_DE_EVENT_IND: + mt7925_nan_mcu_handle_de_event(dev, tlv); + break; + case NAN_UNI_EVENT_REPORT_DW_END: + mt7925_nan_handle_dw_ind(dev, tlv); + break; + default: + break; + } + + tlv_len -= len; + tlv = (struct tlv *)((u8 *)tlv + len); + } +} + +static int mt7925_nan_avail_ctrl_tlv(struct sk_buff *skb, + struct ieee80211_vif *vif) +{ + struct mt7925_nan_avail_ctrl_tlv *avail_ctrl_tlv; + struct ieee80211_nan_sched_cfg *sched; + struct tlv *tlv; + u8 seq_id = 0; + u16 ctrl = 0; + + if (!skb || !vif) + return -EINVAL; + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_UPDATE_AVAILABILITY_CTRL, + sizeof(struct mt7925_nan_avail_ctrl_tlv)); + + if (!tlv) + return -ENOMEM; + + sched = &vif->cfg.nan_sched; + + ctrl = mt7925_nan_avail_attr_ctrl(sched); + if (sched->avail_blob_len >= NAN_AVAIL_ATTR_CTRL_OFFSET + 2) + seq_id = sched->avail_blob[NAN_AVAIL_SEQ_ID_OFFSET]; + + avail_ctrl_tlv = (struct mt7925_nan_avail_ctrl_tlv *)tlv; + avail_ctrl_tlv->avail_ctrl = + cpu_to_le16(ctrl & NAN_AVAIL_CTRL_CHECK_FOR_CHANGED); + avail_ctrl_tlv->seq_id = seq_id; + + return 0; +} + +static u32 mt7925_nan_slot_to_bitmap(struct ieee80211_vif *vif, + struct mt7925_nan_ch_timeline *ch_list) +{ + struct ieee80211_nan_channel **slots = vif->cfg.nan_sched.schedule; + struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; + u32 num_channels = 0; + u32 i, j; + + for (i = 0; i < ARRAY_SIZE(mvif->nan.local_sched); i++) { + struct cfg80211_chan_def *slot_chan = &mvif->nan.local_sched[i]; + struct ieee80211_nan_channel *slot = slots[i]; + bool is_found = false; + + if (slot && !IS_ERR(slot) && slot->chanctx_conf) { + *slot_chan = slot->chanctx_conf->def; + } else { + memset(slot_chan, 0, sizeof(*slot_chan)); + continue; + } + + for (j = 0; j < num_channels; j++) { + u32 raw = le32_to_cpu(ch_list[j].ch_info); + + if (FIELD_GET(NAN_CH_CTRL_PRIMARY_CH, raw) == + slot_chan->chan->hw_value) { + u32 map = le32_to_cpu(ch_list[j].avail_map[0]); + + ch_list[j].avail_map[0] = cpu_to_le32(map | BIT(i)); + le32_add_cpu(&ch_list[j].num, 1); + is_found = true; + break; + } + } + + if (!is_found && num_channels < NAN_TIMELINE_MGMT_CHNL_LIST_NUM) { + ch_list[num_channels].ch_info = + cpu_to_le32(FIELD_PREP(NAN_CH_CTRL_OP_CLASS, + slot->channel_entry[0]) | + FIELD_PREP(NAN_CH_CTRL_PRIMARY_CH, + slot_chan->chan->hw_value)); + ch_list[num_channels].avail_map[0] = cpu_to_le32(BIT(i)); + le32_add_cpu(&ch_list[num_channels].num, 1); + ch_list[num_channels].is_valid++; + num_channels++; + } + } + + return num_channels; +} + +static int mt7925_nan_avail_tlv(struct sk_buff *skb, + struct ieee80211_vif *vif) +{ + struct mt7925_nan_avail_entry_tlv *avail_tlv; + struct ieee80211_nan_sched_cfg *sched; + struct tlv *tlv; + u16 ctrl = 0; + + if (!skb || !vif) + return -EINVAL; + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_UPDATE_AVAILABILITY, + sizeof(struct mt7925_nan_avail_entry_tlv)); + + if (!tlv) + return -ENOMEM; + + sched = &vif->cfg.nan_sched; + + ctrl = mt7925_nan_avail_attr_ctrl(sched); + + avail_tlv = (struct mt7925_nan_avail_entry_tlv *)tlv; + avail_tlv->map_id = ctrl & NAN_AVAIL_CTRL_MAPID; + avail_tlv->is_cond_avail = false; + avail_tlv->timeline_idx = 0; + + mt7925_nan_slot_to_bitmap(vif, avail_tlv->ch_list); + + avail_tlv->is_multi_map = false; + + return 0; +} + +void mt7925_nan_local_sched_changed(struct mt792x_dev *dev, + struct ieee80211_vif *vif) +{ + struct mt7925_nan_common_hdr *hdr; + struct mt76_dev *mdev; + struct sk_buff *skb; + + if (!dev || !vif) + return; + + mdev = &dev->mt76; + + skb = mt76_mcu_msg_alloc(mdev, NULL, MT7925_NAN_AVAIL_MAX_SIZE); + if (!skb) + return; + + hdr = (struct mt7925_nan_common_hdr *)skb_put(skb, sizeof(*hdr)); + memset(hdr, 0, sizeof(*hdr)); + + if (mt7925_nan_avail_ctrl_tlv(skb, vif) || + mt7925_nan_avail_tlv(skb, vif)) { + dev_kfree_skb(skb); + return; + } + + mt76_mcu_skb_send_msg(mdev, skb, + MCU_UNI_CMD(NAN), true); +} + +static int mt7925_nan_peer_rec_tlv(struct sk_buff *skb, + struct ieee80211_sta *sta, + struct mt792x_sta *msta, + u8 is_activate) +{ + struct mt7925_nan_sched_manage_peer_rec_tlv *peer_rec_tlv; + struct tlv *tlv; + + if (!skb || !sta || !msta) + return -EINVAL; + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_MANAGE_PEER_SCH_RECORD, + sizeof(struct mt7925_nan_sched_manage_peer_rec_tlv)); + + if (!tlv) + return -ENOMEM; + + peer_rec_tlv = (struct mt7925_nan_sched_manage_peer_rec_tlv *)tlv; + peer_rec_tlv->sch_idx = cpu_to_le32(msta->nan_sched.sch_idx); + peer_rec_tlv->is_activate = is_activate; + memcpy(peer_rec_tlv->nmi_addr, sta->addr, ETH_ALEN); + + return 0; +} + +static int mt7925_nan_peer_cap_tlv(struct sk_buff *skb, + struct ieee80211_sta *sta, + struct mt792x_sta *msta) +{ + struct mt7925_nan_sched_update_peer_cap_tlv *peer_cap_tlv; + struct ieee80211_nan_peer_sched *sched; + enum nl80211_band band; + struct tlv *tlv; + u16 primary_ch; + u32 i; + + if (!skb || !sta || !msta) + return -EINVAL; + + sched = sta->nan_sched; + if (!sched) + return -EINVAL; + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_UPDATE_PEER_CAPABILITY, + sizeof(struct mt7925_nan_sched_update_peer_cap_tlv)); + + if (!tlv) + return -ENOMEM; + + peer_cap_tlv = (struct mt7925_nan_sched_update_peer_cap_tlv *)tlv; + peer_cap_tlv->sch_idx = cpu_to_le32(msta->nan_sched.sch_idx); + peer_cap_tlv->supported_bands = BIT(NAN_SUPPORTED_BAND_ID_2P4G); + peer_cap_tlv->max_chnl_switch_time = cpu_to_le16(sched->max_chan_switch); + + for (i = 0; i < sched->n_channels; i++) { + if (!sched->channels[i].chanctx_conf) + continue; + + band = sched->channels[i].chanctx_conf->def.chan->band; + primary_ch = + sched->channels[i].chanctx_conf->def.chan->hw_value; + + if (band == NL80211_BAND_2GHZ) + peer_cap_tlv->peer_supported_bands |= + BIT(NAN_SUPPORTED_BN_2G); + else if (primary_ch >= UNII1_LOWER_BOUND && + primary_ch <= UNII1_UPPER_BOUND) + peer_cap_tlv->peer_supported_bands |= + BIT(NAN_SUPPORTED_BN_5G_LOW); + else if (primary_ch >= UNII3_LOWER_BOUND && + primary_ch <= UNII3_UPPER_BOUND) + peer_cap_tlv->peer_supported_bands |= + BIT(NAN_SUPPORTED_BN_5G_HIGH); + } + + return 0; +} + +static void +mt7925_nan_fill_crb_committed(struct mt7925_nan_sched_update_crb_tlv *crb_tlv, + struct ieee80211_nan_peer_sched *sched) +{ + u32 m, slot; + + if (!sched) + return; + + for (m = 0; m < CFG80211_NAN_MAX_PEER_MAPS && + m < NAN_TIMELINE_MGMT_SIZE; m++) { + struct mt7925_nan_sched_timeline *tl = + &crb_tlv->comm_faw_timeline[m]; + struct ieee80211_nan_peer_map *map = &sched->maps[m]; + + if (map->map_id == CFG80211_NAN_INVALID_MAP_ID) + continue; + + tl->map_id = map->map_id; + + /* + * Convert peer schedule slots to FW avail_map bitmap. + * Each bit in avail_map[0] represents one time slot where + * the peer has committed availability. + */ + for (slot = 0; slot < CFG80211_NAN_SCHED_NUM_TIME_SLOTS; + slot++) { + struct ieee80211_nan_channel *ch = map->slots[slot]; + + if (!ch || !ch->chanctx_conf) + continue; + + tl->avail_map[0] |= cpu_to_le32(BIT(slot)); + } + } +} + +static int mt7925_nan_update_crb_tlv(struct sk_buff *skb, + struct ieee80211_sta *sta, + struct mt792x_sta *msta) +{ + struct mt7925_nan_sched_update_crb_tlv *crb_tlv; + struct tlv *tlv; + + if (!skb || !sta || !msta) + return -EINVAL; + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_UPDATE_CRB, + sizeof(struct mt7925_nan_sched_update_crb_tlv)); + + if (!tlv) + return -ENOMEM; + + crb_tlv = (struct mt7925_nan_sched_update_crb_tlv *)tlv; + crb_tlv->sch_idx = cpu_to_le32(msta->nan_sched.sch_idx); + crb_tlv->flags = NAN_CRB_USE_DATA_PATH; + crb_tlv->is_use_ranging = false; + crb_tlv->comm_ndc_ctrl.is_valid = false; + + mt7925_nan_fill_crb_committed(crb_tlv, sta->nan_sched); + + return 0; +} + +int mt792x_nan_set_peer_schedule(struct mt792x_dev *dev, + struct ieee80211_sta *sta) +{ + struct mt7925_nan_common_hdr *hdr; + struct mt792x_sta *msta; + struct mt792x_nan *nan; + struct mt76_dev *mdev; + struct sk_buff *skb; + + if (!dev || !sta) + return -EINVAL; + + mdev = &dev->mt76; + + skb = mt76_mcu_msg_alloc(mdev, NULL, MT7925_NAN_PEER_MAX_SIZE); + if (!skb) + return -ENOMEM; + + hdr = (struct mt7925_nan_common_hdr *)skb_put(skb, sizeof(*hdr)); + memset(hdr, 0, sizeof(*hdr)); + + msta = (struct mt792x_sta *)sta->drv_priv; + nan = &msta->vif->nan; + + /* Allocate connection index on first call for this peer */ + if (!msta->nan_sched.idx_assigned) { + int idx = find_first_zero_bit(&nan->conn_bitmap, + NAN_MAX_CONN_CFG); + if (idx >= NAN_MAX_CONN_CFG) { + dev_kfree_skb(skb); + return -ENOSPC; + } + + set_bit(idx, &nan->conn_bitmap); + msta->nan_sched.sch_idx = idx; + msta->nan_sched.idx_assigned = true; + + if (mt7925_nan_peer_rec_tlv(skb, sta, msta, true) || + mt7925_nan_peer_cap_tlv(skb, sta, msta)) { + dev_kfree_skb(skb); + return -ENOMEM; + } + } + + if (mt7925_nan_update_crb_tlv(skb, sta, msta)) { + dev_kfree_skb(skb); + return -ENOMEM; + } + + return mt76_mcu_skb_send_msg(mdev, skb, + MCU_UNI_CMD(NAN), true); +} + +int mt792x_nan_set_peer_rec(struct mt76_dev *mdev, + struct ieee80211_sta *sta) +{ + struct mt7925_nan_common_hdr *hdr; + struct mt792x_sta *msta; + struct mt792x_nan *nan; + struct sk_buff *skb; + + if (!mdev || !sta) + return -EINVAL; + + skb = mt76_mcu_msg_alloc(mdev, NULL, + sizeof(struct mt7925_nan_common_hdr) + + sizeof(struct mt7925_nan_sched_manage_peer_rec_tlv)); + if (!skb) + return -ENOMEM; + + hdr = (struct mt7925_nan_common_hdr *)skb_put(skb, sizeof(*hdr)); + memset(hdr, 0, sizeof(*hdr)); + + msta = (struct mt792x_sta *)sta->drv_priv; + nan = &msta->vif->nan; + + if (!msta->nan_sched.idx_assigned) { + dev_kfree_skb(skb); + return 0; + } + + if (mt7925_nan_peer_rec_tlv(skb, sta, msta, false)) { + dev_kfree_skb(skb); + return -ENOMEM; + } + + clear_bit(msta->nan_sched.sch_idx, &nan->conn_bitmap); + msta->nan_sched.idx_assigned = false; + + return mt76_mcu_skb_send_msg(mdev, skb, + MCU_UNI_CMD(NAN), true); +} + +int mt792x_nan_map_sta_rec(struct mt76_dev *mdev, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta) +{ + struct mt7925_nan_sched_map_sta_rec_tlv *map_tlv; + struct mt7925_nan_common_hdr *hdr; + struct ieee80211_sta *nmi_sta; + struct mt792x_sta *nmi_msta; + struct mt792x_sta *msta; + u8 nmi_addr[ETH_ALEN]; + struct sk_buff *skb; + int ndp_ctx_id = 0; + struct tlv *tlv; + + if (!mdev || !vif || !sta) + return -EINVAL; + + msta = (struct mt792x_sta *)sta->drv_priv; + + rcu_read_lock(); + nmi_sta = rcu_dereference(sta->nmi); + if (!nmi_sta) { + rcu_read_unlock(); + dev_err(mdev->dev, "NAN: NMI sta not found for NDI sta %pM\n", + sta->addr); + return -EINVAL; + } + + memcpy(nmi_addr, nmi_sta->addr, ETH_ALEN); + nmi_msta = (struct mt792x_sta *)nmi_sta->drv_priv; + + ndp_ctx_id = find_first_zero_bit(&nmi_msta->nan_sched.ndp_ctx_bitmap, + NAN_MAX_NDP_CXT); + if (ndp_ctx_id < NAN_MAX_NDP_CXT) + set_bit(ndp_ctx_id, &nmi_msta->nan_sched.ndp_ctx_bitmap); + else + ndp_ctx_id = 0; + rcu_read_unlock(); + + msta->nan_sched.ndp_ctx_id = ndp_ctx_id; + + skb = mt76_mcu_msg_alloc(mdev, NULL, + sizeof(struct mt7925_nan_common_hdr) + + sizeof(struct mt7925_nan_sched_map_sta_rec_tlv)); + if (!skb) + return -ENOMEM; + + hdr = (struct mt7925_nan_common_hdr *)skb_put(skb, sizeof(*hdr)); + memset(hdr, 0, sizeof(*hdr)); + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_MAP_STA_RECORD, + sizeof(struct mt7925_nan_sched_map_sta_rec_tlv)); + if (!tlv) { + dev_kfree_skb(skb); + return -ENOMEM; + } + + map_tlv = (struct mt7925_nan_sched_map_sta_rec_tlv *)tlv; + memcpy(map_tlv->nmi_addr, nmi_addr, ETH_ALEN); + map_tlv->sta_rec_idx = msta->deflink.wcid.idx; + map_tlv->ndp_ctx_id = ndp_ctx_id; + map_tlv->role_idx = 0; + memcpy(map_tlv->ndi_addr, vif->addr, ETH_ALEN); + + return mt76_mcu_skb_send_msg(mdev, skb, + MCU_UNI_CMD(NAN), true); +} diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/nan.h b/drivers/net/wireless/mediatek/mt76/mt7925/nan.h new file mode 100644 index 00000000000000..356d9ef7f66436 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7925/nan.h @@ -0,0 +1,419 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* Copyright (C) 2025-2026 MediaTek Inc. */ + +#ifndef __MT7925_NAN_H +#define __MT7925_NAN_H + +#include +#include + +#include "../mt76_connac_mcu.h" + +#define NAN_MAX_SOCIAL_CHANNELS 3 +#define NAN_ANCHOR_MASTER_RANK_NUM 8 +#define NAN_5G_LOW_DISC_CHANNEL 44 +#define NAN_5G_HIGH_DISC_CHANNEL 149 +#define NAN_MAX_MASTER_PREFERENCE 255 +#define NAN_DEFAULT_DW_INTERVAL 1 +#define NAN_DEFAULT_DISC_BCN_INTERVAL 100 +#define NAN_TOTAL_DW 16 +#define NAN_SUPPORTED_2G_FAW_CH_NUM 4 +#define NAN_SUPPORTED_5G_FAW_CH_NUM 4 +#define NAN_TIMELINE_MGMT_SIZE 2 +#define NAN_TIMELINE_MGMT_CHNL_LIST_NUM \ + ((NAN_SUPPORTED_2G_FAW_CH_NUM + \ + NAN_SUPPORTED_5G_FAW_CH_NUM) / NAN_TIMELINE_MGMT_SIZE) +#define NAN_NUM_AVAIL_DB 2 +#define NAN_NDC_ATTRIBUTE_ID_LENGTH 6 +#define NAN_MAX_CONN_CFG 8 +#define NAN_MAX_NDP_CXT 4 + +#define MT7925_NAN_CONF_MAX_SIZE \ + (sizeof(struct mt7925_nan_common_hdr) + \ + sizeof(struct mt7925_nan_master_preference_tlv) + \ + sizeof(struct mt7925_nan_dw_interval_tlv) + \ + sizeof(struct mt7925_nan_cluster_id_tlv) + \ + sizeof(struct mt7925_nan_sync_rssi_tlv)) + +#define MT7925_NAN_AVAIL_MAX_SIZE \ + (sizeof(struct mt7925_nan_common_hdr) + \ + sizeof(struct mt7925_nan_avail_ctrl_tlv) + \ + sizeof(struct mt7925_nan_avail_entry_tlv)) + +#define MT7925_NAN_PEER_MAX_SIZE \ + (sizeof(struct mt7925_nan_common_hdr) + \ + sizeof(struct mt7925_nan_sched_manage_peer_rec_tlv) + \ + sizeof(struct mt7925_nan_sched_update_peer_cap_tlv) + \ + sizeof(struct mt7925_nan_sched_update_crb_tlv)) + +/* NAN Availability Attribute */ +#define NAN_AVAIL_ATTR_ID_OFFSET 0 +#define NAN_AVAIL_ATTR_LEN_OFFSET 1 +#define NAN_AVAIL_SEQ_ID_OFFSET 3 +#define NAN_AVAIL_ATTR_CTRL_OFFSET 4 + +/* NAN Availability Attribute - Attribute Control Field */ +#define NAN_AVAIL_CTRL_MAPID GENMASK(3, 0) +#define NAN_AVAIL_CTRL_COMMIT_CHANGED BIT(4) +#define NAN_AVAIL_CTRL_POTN_CHANGED BIT(5) +#define NAN_AVAIL_CTRL_PUBLIC_AVAIL_CHANGED BIT(6) +#define NAN_AVAIL_CTRL_NDC_CHANGED BIT(7) +#define NAN_AVAIL_CTRL_CHECK_FOR_CHANGED GENMASK(7, 4) + +#define UNII1_LOWER_BOUND 36 +#define UNII1_UPPER_BOUND 50 +#define UNII3_LOWER_BOUND 149 +#define UNII3_UPPER_BOUND 165 + +enum nan_uni_cmd_tag { + NAN_UNI_CMD_SET_MASTER_PREFERENCE = 0, + NAN_UNI_CMD_ENABLE_REQUEST = 7, + NAN_UNI_CMD_DISABLE_REQUEST = 8, + NAN_UNI_CMD_UPDATE_AVAILABILITY = 9, + NAN_UNI_CMD_UPDATE_CRB = 10, + NAN_UNI_CMD_MANAGE_PEER_SCH_RECORD = 12, + NAN_UNI_CMD_MAP_STA_RECORD = 13, + NAN_UNI_CMD_UPDATE_AVAILABILITY_CTRL = 20, + NAN_UNI_CMD_UPDATE_PEER_CAPABILITY = 21, + NAN_UNI_CMD_CHANGE_NMI_ADDRESS = 24, + NAN_UNI_CMD_SET_DW_INTERVAL = 26, + NAN_UNI_CMD_SET_SYNC_RSSI = 39, + NAN_UNI_CMD_SET_CLUSTER_ID = 40, + NAN_UNI_CMD_KEY_MANAGEMENT = 53, +}; + +enum nan_uni_event_tag { + NAN_UNI_EVENT_ID_DE_EVENT_IND = 19, + NAN_UNI_EVENT_REPORT_DW_END = 60, +}; + +enum nan_disc_event_type { + NAN_EVENT_ID_DISC_MAC_ADDR = 0, + NAN_EVENT_ID_JOINED_CLUSTER = 2, +}; + +/* NAN 4.0 Table 79. Device Capability attribute format, Supported Bands */ +enum nan_supported_bands { + NAN_SUPPORTED_BAND_ID_2P4G = 2, + NAN_SUPPORTED_BAND_ID_5G = 4, + NAN_PROPRIETARY_BAND_ID_6G = 6, + NAN_SUPPORTED_BAND_ID_6G = 7, +}; + +enum nan_peer_supported_bands { + NAN_SUPPORTED_BN_2G = 0, + NAN_SUPPORTED_BN_5G_LOW, + NAN_SUPPORTED_BN_5G_HIGH, + NAN_SUPPORTED_BN_6G, + NAN_SUPPORTED_BN_NUM +}; + +#define NAN_CH_CTRL_OP_CLASS GENMASK(15, 8) +#define NAN_CH_CTRL_PRIMARY_CH GENMASK(23, 16) + +#define NAN_CRB_USE_DATA_PATH BIT(0) +#define NAN_CRB_AVAIL_6G_FORMAT GENMASK(2, 1) + +struct mt7925_nan_social_ch_scan_params { + u8 dwell_time[NAN_MAX_SOCIAL_CHANNELS]; + __le16 scan_period[NAN_MAX_SOCIAL_CHANNELS]; +} __packed; + +/* Firmware-reported NAN device information */ +struct nan_dev_info_evt { + u8 is_enabled; + u8 my_addr[ETH_ALEN]; + u8 en_fw_election; + __le32 nan_dev_role; + __le32 nan_dev_state; + u8 mst_preference; + u8 random_factor; + u8 cnt_hop; + u8 cluster_id[ETH_ALEN]; + u8 anchor_mst_addr[ETH_ALEN]; + u8 am_preference; + u8 am_random_factor; + u8 parent_mac[ETH_ALEN]; + u8 parent_am_preference; + u8 parent_am_factor; + __le32 ambtt; + __le32 tsf[2]; + u8 pn_igtk[6]; + u8 pn_bigtk[6]; +}; + +/* Firmware NAN discovery window event */ +struct nan_rpt_dw_evt { + struct nan_dev_info_evt device_info; + __le32 expected_tsf_h; + __le32 expected_tsf_l; + __le32 actual_tsf_h; + __le32 actual_tsf_l; + __le16 channel; + __le16 dw_num; +}; + +struct mt7925_nan_conf_dw { + u8 config_2dot4g_dw_band; + __le32 dw_2dot4g_interval_val; + + u8 config_5g_dw_band; + __le32 dw_5g_interval_val; +} __packed; + +struct mt7925_nan_enable_req_tlv { + __le16 tag; + __le16 len; + + u8 master_pref; + __le16 cluster_low; + __le16 cluster_high; + + u8 config_support_5g; + u8 support_5g_val; + + u8 config_sid_beacon; + u8 sid_beacon_val; + + u8 config_2dot4g_rssi_close; + u8 rssi_close_2dot4g_val; + u8 config_2dot4g_rssi_middle; + u8 rssi_middle_2dot4g_val; + + u8 config_2dot4g_rssi_proximity; + u8 rssi_proximity_2dot4g_val; + u8 config_hop_count_limit; + u8 hop_count_limit_val; + + u8 config_2dot4g_support; + u8 support_2dot4g_val; + + u8 config_2dot4g_beacons; + u8 beacon_2dot4g_val; + + u8 config_2dot4g_sdf; + u8 sdf_2dot4g_val; + + u8 config_5g_beacons; + u8 beacon_5g_val; + + u8 config_5g_sdf; + u8 sdf_5g_val; + + u8 config_5g_rssi_close; + u8 rssi_close_5g_val; + + u8 config_5g_rssi_middle; + u8 rssi_middle_5g_val; + + u8 config_5g_rssi_close_proximity; + u8 rssi_close_proximity_5g_val; + + u8 config_rssi_window_size; + u8 rssi_window_size_val; + + u8 config_oui; + __le32 oui_val; + + u8 config_intf_addr; + u8 intf_addr_val[ETH_ALEN]; + + u8 config_cluster_attribute_val; + + u8 config_scan_params; + struct mt7925_nan_social_ch_scan_params scan_params_val; + + u8 config_random_factor_force; + u8 random_factor_force_val; + + u8 config_hop_count_force; + u8 hop_count_force_val; + + u8 config_24g_channel; + __le32 channel_24g_val; + + u8 config_5g_channel; + __le32 channel_5g_val; + + struct mt7925_nan_conf_dw config_dw; + + u8 config_disc_mac_addr_randomization; + __le32 disc_mac_addr_rand_interval_sec; + + u8 discovery_indication_cfg; + + u8 config_subscribe_sid_beacon; + __le32 subscribe_sid_beacon_val; + + u8 enable_log_slot_statistics; +} __packed __aligned(4); + +struct mt7925_nan_common_hdr { + u8 reserved[4]; +}; + +struct mt7925_nan_master_preference_tlv { + __le16 tag; + __le16 len; + u8 master_preference; + u8 reserved[3]; +} __packed __aligned(4); + +struct mt7925_nan_dw_interval_tlv { + __le16 tag; + __le16 len; + u8 dw_interval; + u8 vendor_ioctl; + __le16 disc_bcn_interval; +} __packed __aligned(4); + +struct mt7925_nan_cluster_id_tlv { + __le16 tag; + __le16 len; + u8 cluster_id[ETH_ALEN]; + u8 reserved[2]; +} __packed __aligned(4); + +struct mt7925_nan_sync_rssi_tlv { + __le16 tag; + __le16 len; + s8 rssi_close_2g; + s8 rssi_middle_2g; + s8 rssi_close_5g; + s8 rssi_middle_5g; +} __packed __aligned(4); + +struct mt7925_nan_de_event { + u8 event_type; + u8 cluster_id[ETH_ALEN]; + u8 anchor_master_rank[NAN_ANCHOR_MASTER_RANK_NUM]; + u8 own_nmi[ETH_ALEN]; + u8 master_nmi[ETH_ALEN]; +}; + +struct mt7925_nan_nmi_addr_tlv { + __le16 tag; + __le16 len; + u8 nmi_addr[ETH_ALEN]; +} __packed __aligned(4); + +struct mt7925_nan_avail_ctrl_tlv { + __le16 tag; + __le16 len; + __le16 avail_ctrl; + u8 seq_id; + u8 reserved[1]; +} __packed __aligned(4); + +struct mt7925_nan_ch_timeline { + u8 is_valid; + u8 reserved[3]; + + __le32 ch_info; + + __le32 num; + __le32 avail_map[NAN_TOTAL_DW]; +}; + +struct mt7925_nan_avail_entry_tlv { + __le16 tag; + __le16 len; + u8 map_id; + u8 is_cond_avail; + u8 timeline_idx; + u8 is_multi_map; + + struct mt7925_nan_ch_timeline ch_list[NAN_TIMELINE_MGMT_CHNL_LIST_NUM]; +} __packed __aligned(4); + +struct mt7925_nan_sched_manage_peer_rec_tlv { + __le16 tag; + __le16 len; + __le32 sch_idx; + u8 is_activate; + u8 nmi_addr[ETH_ALEN]; + u8 reserved[1]; +} __packed __aligned(4); + +struct mt7925_nan_sched_update_peer_cap_tlv { + __le16 tag; + __le16 len; + __le32 sch_idx; + u8 supported_bands; + __le16 max_chnl_switch_time; + u8 peer_supported_bands; +} __packed __aligned(4); + +struct mt7925_nan_sched_timeline { + u8 map_id; + u8 local_map_id; + u8 reserved[2]; + union { + __le32 avail_map[NAN_TOTAL_DW]; + u8 avail_block[NAN_TOTAL_DW * 4]; + }; +}; + +struct mt7925_nan_sched_faw_ndc_timeline { + __le32 avail_map[NAN_TOTAL_DW]; +}; + +struct mt7925_nan_sched_ndc_ctrl { + u8 is_valid; + u8 ndc_id[NAN_NDC_ATTRIBUTE_ID_LENGTH]; + u8 ndc_idx; + struct mt7925_nan_sched_timeline timeline[NAN_NUM_AVAIL_DB]; +}; + +struct mt7925_nan_sched_update_crb_tlv { + __le16 tag; + __le16 len; + __le32 sch_idx; + u8 flags; + u8 is_use_ranging; + u8 reserved[2]; + struct mt7925_nan_sched_timeline comm_ranging_timeline[NAN_TIMELINE_MGMT_SIZE]; + struct mt7925_nan_sched_timeline comm_faw_timeline[NAN_TIMELINE_MGMT_SIZE]; + struct mt7925_nan_sched_ndc_ctrl comm_ndc_ctrl; + struct mt7925_nan_sched_faw_ndc_timeline faw_ndc_timeline[NAN_TIMELINE_MGMT_SIZE]; +} __packed __aligned(4); + +struct mt7925_nan_sched_map_sta_rec_tlv { + __le16 tag; + __le16 len; + u8 nmi_addr[ETH_ALEN]; + u8 sta_rec_idx; + u8 ndp_ctx_id; + + __le32 role_idx; + u8 ndi_addr[ETH_ALEN]; + u8 reserved[2]; +} __packed __aligned(4); + +int mt7925_nan_enable(struct ieee80211_vif *vif, + struct mt792x_dev *dev, + struct cfg80211_nan_conf *conf); + +int mt7925_nan_disable(struct ieee80211_vif *vif, + struct mt792x_dev *dev); + +int mt7925_nan_change_configure(struct ieee80211_vif *vif, + struct mt792x_dev *dev, + struct cfg80211_nan_conf *conf); + +void mt7925_nan_mcu_event(struct mt792x_dev *dev, struct sk_buff *skb); + +void mt7925_nan_local_sched_changed(struct mt792x_dev *dev, + struct ieee80211_vif *vif); + +int mt792x_nan_set_peer_schedule(struct mt792x_dev *dev, + struct ieee80211_sta *sta); + +int mt792x_nan_set_peer_rec(struct mt76_dev *mdev, + struct ieee80211_sta *sta); + +int mt792x_nan_map_sta_rec(struct mt76_dev *mdev, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta); + +#endif diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/regd.c b/drivers/net/wireless/mediatek/mt76/mt7925/regd.c index 16f56ee879d453..0235437d11d591 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/regd.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/regd.c @@ -217,6 +217,36 @@ mt7925_regd_is_valid_alpha2(const char *alpha2) return false; } +bool +mt7925_regd_is_valid_channel(struct mt792x_dev *dev, + enum nl80211_band band, + struct ieee80211_channel *chan) +{ + struct ieee80211_hw *hw = mt76_hw(dev); + struct wiphy *wiphy = hw->wiphy; + struct ieee80211_supported_band *sband; + struct ieee80211_channel *ch; + int i; + + if (!chan) + return false; + + sband = wiphy->bands[band]; + if (!sband) + return false; + + for (i = 0; i < sband->n_channels; i++) { + ch = &sband->channels[i]; + + if (ch->hw_value == chan->hw_value && + ((ch->flags & IEEE80211_CHAN_DISABLED) == 0)) + return true; + } + + return false; +} +EXPORT_SYMBOL_GPL(mt7925_regd_is_valid_channel); + int mt7925_regd_change(struct mt792x_phy *phy, char *alpha2) { struct wiphy *wiphy = phy->mt76->hw->wiphy; diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/regd.h b/drivers/net/wireless/mediatek/mt76/mt7925/regd.h index 0767f078862e70..0b0754cf8ae70a 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/regd.h +++ b/drivers/net/wireless/mediatek/mt76/mt7925/regd.h @@ -13,6 +13,9 @@ void mt7925_regd_be_ctrl(struct mt792x_dev *dev, u8 *alpha2); void mt7925_regd_notifier(struct wiphy *wiphy, struct regulatory_request *req); bool mt7925_regd_clc_supported(struct mt792x_dev *dev); int mt7925_regd_change(struct mt792x_phy *phy, char *alpha2); +bool mt7925_regd_is_valid_channel(struct mt792x_dev *dev, + enum nl80211_band band, + struct ieee80211_channel *chan); int mt7925_regd_init(struct mt792x_phy *phy); #endif diff --git a/drivers/net/wireless/mediatek/mt76/mt792x.h b/drivers/net/wireless/mediatek/mt76/mt792x.h index 70073b43af5431..89c3f84a776a0f 100644 --- a/drivers/net/wireless/mediatek/mt76/mt792x.h +++ b/drivers/net/wireless/mediatek/mt76/mt792x.h @@ -115,6 +115,18 @@ struct mt792x_link_sta { struct ieee80211_link_sta *pri_link; }; +struct mt792x_sta_nan_sched { + u16 committed_dw; + u32 sch_idx; + bool idx_assigned; + unsigned long ndp_ctx_bitmap; + u8 ndp_ctx_id; /* assigned NDP context ID (for NDI sta) */ + struct { + u8 map_id; + struct cfg80211_chan_def chans[CFG80211_NAN_SCHED_NUM_TIME_SLOTS]; + } maps[CFG80211_NAN_MAX_PEER_MAPS]; +}; + struct mt792x_sta { struct mt792x_link_sta deflink; /* must be first */ struct mt792x_link_sta __rcu *link[IEEE80211_MLD_MAX_NUM_LINKS]; @@ -123,6 +135,9 @@ struct mt792x_sta { u16 valid_links; u8 deflink_id; + + /* NAN peer schedule */ + struct mt792x_sta_nan_sched nan_sched; }; DECLARE_EWMA(rssi, 10, 8); @@ -139,6 +154,25 @@ struct mt792x_bss_conf { unsigned int link_id; }; +struct mt792x_nan_conf { + u8 master_pref; + u8 bands; + u8 cluster_id[ETH_ALEN]; + u32 discovery_beacon_interval; + bool enable_dw_notification; +}; + +struct mt792x_nan { + struct mt792x_nan_conf conf; + + /* Scheduler */ + struct cfg80211_chan_def local_sched[CFG80211_NAN_SCHED_NUM_TIME_SLOTS]; + u32 seq_id; + + /* Connection index bitmap, up to NAN_MAX_CONN_CFG peers */ + unsigned long conn_bitmap; +}; + struct mt792x_vif { struct mt792x_bss_conf bss_conf; /* must be first */ struct mt792x_bss_conf __rcu *link_conf[IEEE80211_MLD_MAX_NUM_LINKS]; @@ -153,6 +187,8 @@ struct mt792x_vif { struct work_struct csa_work; struct timer_list csa_timer; + + struct mt792x_nan nan; }; struct mt792x_phy { @@ -283,6 +319,8 @@ struct mt792x_dev { u32 backup_l2; struct ieee80211_chanctx_conf *new_ctx; + + struct ieee80211_vif *nan_vif; }; static inline struct mt792x_bss_conf * diff --git a/drivers/net/wireless/mediatek/mt76/mt792x_usb.c b/drivers/net/wireless/mediatek/mt76/mt792x_usb.c index 910132e94956aa..47f80c9ec4e7de 100644 --- a/drivers/net/wireless/mediatek/mt76/mt792x_usb.c +++ b/drivers/net/wireless/mediatek/mt76/mt792x_usb.c @@ -31,10 +31,75 @@ static void mt792xu_reset_work(struct work_struct *work) atomic_set(&dev->usb_reset_pending, 0); } +static void mt792xu_queue_usb_reset(struct mt792x_dev *dev, int err) +{ + if (!atomic_xchg(&dev->usb_reset_pending, 1)) { + dev_warn(dev->mt76.dev, + "USB transport access failed (%d), queueing device reset\n", + err); + + schedule_work(&dev->usb_reset_work); + } +} + +static u32 mt792xu_bus_hung_rr(struct mt76_dev *mdev, u32 offset) +{ + return 0; +} + +static void mt792xu_bus_hung_wr(struct mt76_dev *mdev, u32 offset, u32 val) +{ +} + +static u32 mt792xu_bus_hung_rmw(struct mt76_dev *mdev, u32 offset, + u32 mask, u32 val) +{ + return 0; +} + +static void mt792xu_bus_hung_write_copy(struct mt76_dev *mdev, u32 offset, + const void *data, int len) +{ +} + +static void mt792xu_bus_hung_read_copy(struct mt76_dev *mdev, u32 offset, + void *data, int len) +{ + memset(data, 0, len); +} + +static const struct mt76_bus_ops mt792xu_bus_hung_ops = { + .rr = mt792xu_bus_hung_rr, + .wr = mt792xu_bus_hung_wr, + .rmw = mt792xu_bus_hung_rmw, + .write_copy = mt792xu_bus_hung_write_copy, + .read_copy = mt792xu_bus_hung_read_copy, + .type = MT76_BUS_USB, +}; + +static void mt792xu_set_bus_hung(struct mt792x_dev *dev) +{ + atomic_set(&dev->mt76.bus_hung, true); + + if (READ_ONCE(dev->mt76.bus) == &mt792xu_bus_hung_ops) + return; + + WRITE_ONCE(dev->mt76.bus, &mt792xu_bus_hung_ops); +} + +static void mt792xu_ctrl_timeout(struct mt76_dev *mdev, int err) +{ + struct mt792x_dev *dev = container_of(mdev, struct mt792x_dev, mt76); + + mt792xu_set_bus_hung(dev); + mt792xu_queue_usb_reset(dev, err); +} + void mt792xu_reset_work_init(struct mt792x_dev *dev) { INIT_WORK(&dev->usb_reset_work, mt792xu_reset_work); atomic_set(&dev->usb_reset_pending, 0); + dev->mt76.usb.ctrl_timeout = mt792xu_ctrl_timeout; } EXPORT_SYMBOL_GPL(mt792xu_reset_work_init); @@ -62,26 +127,23 @@ EXPORT_SYMBOL_GPL(mt792xu_check_bus); int mt792xu_reset_on_bus_error(struct mt792x_dev *dev) { - int err = 0; + int err; - if (!atomic_read(&dev->mt76.bus_hung)) - err = mt792xu_check_bus(dev); + /* Once hung, the no-op bus ops stay installed until the queued USB + * reset re-probes the device. Do not clear bus_hung here, or the caller + * would run a full reset over dropped register I/O and report success. + */ + if (atomic_read(&dev->mt76.bus_hung)) + return -EIO; + err = mt792xu_check_bus(dev); if (err) { - atomic_set(&dev->mt76.bus_hung, true); - - if (!atomic_xchg(&dev->usb_reset_pending, 1)) { - dev_warn(dev->mt76.dev, - "USB transport access failed (%d), queueing device reset\n", - err); - - schedule_work(&dev->usb_reset_work); - } + mt792xu_set_bus_hung(dev); + mt792xu_queue_usb_reset(dev, err); return err; } - atomic_set(&dev->mt76.bus_hung, false); return 0; } EXPORT_SYMBOL_GPL(mt792xu_reset_on_bus_error); @@ -344,6 +406,9 @@ int mt792xu_wfsys_reset(struct mt792x_dev *dev) u32 val; int i; + if (atomic_read(&dev->mt76.bus_hung)) + return -EIO; + mt792xu_epctl_rst_opt(dev, false); val = mt792xu_uhw_rr(&dev->mt76, desc->rst_reg); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/init.c b/drivers/net/wireless/mediatek/mt76/mt7996/init.c index d6f9aa1ab52d12..fa74aba4269037 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/init.c @@ -34,7 +34,6 @@ static const struct ieee80211_iface_combination if_comb_global = { BIT(NL80211_CHAN_WIDTH_40) | BIT(NL80211_CHAN_WIDTH_80) | BIT(NL80211_CHAN_WIDTH_160), - .beacon_int_min_gcd = 100, }; static const struct ieee80211_iface_combination if_comb_global_7992 = { @@ -47,7 +46,6 @@ static const struct ieee80211_iface_combination if_comb_global_7992 = { BIT(NL80211_CHAN_WIDTH_40) | BIT(NL80211_CHAN_WIDTH_80) | BIT(NL80211_CHAN_WIDTH_160), - .beacon_int_min_gcd = 100, }; static const struct ieee80211_iface_limit if_limits[] = { @@ -1561,7 +1559,6 @@ mt7996_init_eht_caps(struct mt7996_phy *phy, enum nl80211_band band, struct ieee80211_sta_eht_cap *eht_cap = &data->eht_cap; struct ieee80211_eht_cap_elem_fixed *eht_cap_elem = &eht_cap->eht_cap_elem; struct ieee80211_eht_mcs_nss_supp *eht_nss = &eht_cap->eht_mcs_nss_supp; - enum nl80211_chan_width width = phy->mt76->chandef.width; int nss = hweight8(phy->mt76->antenna_mask); int sts = hweight16(phy->mt76->chainmask); u8 val; @@ -1637,11 +1634,16 @@ mt7996_init_eht_caps(struct mt7996_phy *phy, enum nl80211_band band, u8_encode_bits(u8_get_bits(1, GENMASK(1, 0)), IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK); - val = width == NL80211_CHAN_WIDTH_320 ? 0xf : - width == NL80211_CHAN_WIDTH_160 ? 0x7 : - width == NL80211_CHAN_WIDTH_80 ? 0x3 : 0x1; - eht_cap_elem->phy_cap_info[6] = - u8_encode_bits(val, IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK); + eht_cap_elem->phy_cap_info[6] = IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK; + if (band != NL80211_BAND_6GHZ) { + eht_cap_elem->phy_cap_info[6] &= + ~IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_320MHZ; + + if (band != NL80211_BAND_5GHZ) + eht_cap_elem->phy_cap_info[6] &= + ~(IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_160MHZ | + IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_80MHZ); + } val = u8_encode_bits(nss, IEEE80211_EHT_MCS_NSS_RX) | u8_encode_bits(nss, IEEE80211_EHT_MCS_NSS_TX); @@ -1799,6 +1801,8 @@ void mt7996_unregister_device(struct mt7996_dev *dev) { cancel_work_sync(&dev->dump_work); cancel_work_sync(&dev->wed_rro.work); + cancel_work_sync(&dev->reset_work); + cancel_work_sync(&dev->rc_work); mt7996_unregister_phy(mt7996_phy3(dev)); mt7996_unregister_phy(mt7996_phy2(dev)); mt7996_unregister_thermal(&dev->phy); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c index 730fd8e2fa0577..85d4adf1c113d9 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c @@ -166,7 +166,7 @@ static void mt7996_mac_sta_poll(struct mt7996_dev *dev) rssi[0] = to_rssi(GENMASK(7, 0), val); rssi[1] = to_rssi(GENMASK(15, 8), val); rssi[2] = to_rssi(GENMASK(23, 16), val); - rssi[3] = to_rssi(GENMASK(31, 14), val); + rssi[3] = to_rssi(GENMASK(31, 24), val); mlink = rcu_dereference(msta->vif->mt76.link[wcid->link_id]); if (mlink) { @@ -294,6 +294,10 @@ mt7996_mac_fill_rx_rate(struct mt7996_dev *dev, dcm = FIELD_GET(MT_PRXV_DCM, v2); bw = FIELD_GET(MT_PRXV_FRAME_MODE, v2); + /* the hardware reports NSTS; report the data NSS for STBC frames */ + if (stbc && nss > 1) + nss >>= 1; + switch (*mode) { case MT_PHY_TYPE_CCK: cck = true; @@ -349,7 +353,7 @@ mt7996_mac_fill_rx_rate(struct mt7996_dev *dev, case IEEE80211_STA_RX_BW_20: break; case IEEE80211_STA_RX_BW_40: - if (*mode & MT_PHY_TYPE_HE_EXT_SU && + if (*mode == MT_PHY_TYPE_HE_EXT_SU && (idx & MT_PRXV_TX_ER_SU_106T)) { status->bw = RATE_INFO_BW_HE_RU; status->he_ru = @@ -441,7 +445,13 @@ mt7996_mac_fill_rx(struct mt7996_dev *dev, enum mt76_rxq_id q, memset(status, 0, sizeof(*status)); band_idx = FIELD_GET(MT_RXD1_NORMAL_BAND_IDX, rxd1); + if (!mt7996_band_valid(dev, band_idx)) + return -EINVAL; + mphy = dev->mt76.phys[band_idx]; + if (!mphy) + return -EINVAL; + phy = mphy->priv; status->phy_idx = mphy->band_idx; @@ -854,6 +864,33 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi, txwi[6] |= cpu_to_le32(MT_TXD6_DIS_MAT); } +/* The WLAN_IDX in the TXD and TXP must belong to the primary or secondary + * link of an MLD station; any other link id can make the firmware spin when + * that link is in powersave. Completion events carry the same index, so the + * wcid used for status tracking and accounting must match it + */ +struct mt76_wcid *mt7996_get_tx_wcid(struct mt76_wcid *wcid) +{ + struct mt7996_sta_link *msta_link; + struct mt7996_sta *msta; + + if (!wcid->sta) + return wcid; + + msta_link = container_of(wcid, struct mt7996_sta_link, wcid); + msta = msta_link->sta; + + if (!msta || wcid->link_id == msta->seclink_id || + wcid->link_id == msta->deflink_id) + return wcid; + + msta_link = mt7996_sta_link(msta, msta->deflink_id); + if (msta_link) + return &msta_link->wcid; + + return wcid; +} + void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi, struct sk_buff *skb, struct mt76_wcid *wcid, struct ieee80211_key_conf *key, int pid, @@ -1030,6 +1067,11 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, IEEE80211_TX_CTRL_MLO_LINK); } + /* non-MLD frames are LINK_UNSPECIFIED; use the wcid's own link */ + if (link_id == IEEE80211_LINK_UNSPECIFIED && + wcid != &dev->mt76.global_wcid) + link_id = wcid->link_id; + if (link_id != wcid->link_id && link_id != IEEE80211_LINK_UNSPECIFIED) { if (msta) { struct mt7996_sta_link *msta_link = @@ -1086,6 +1128,8 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, tx_info->buf[1].len, DMA_TO_DEVICE); } + wcid = mt7996_get_tx_wcid(wcid); + pid = mt76_tx_status_skb_add(mdev, wcid, tx_info->skb); memset(txwi_ptr, 0, MT_TXD_SIZE); /* Transmit non qos data by 802.11 header and need to fill txd by host*/ @@ -2407,6 +2451,7 @@ mt7996_mac_reset_vif_iter(void *data, u8 *mac, struct ieee80211_vif *vif) rcu_assign_pointer(mvif->link[i], NULL); kfree_rcu(mlink, rcu_head); } + mvif->valid_links = 0; rcu_read_unlock(); } @@ -2420,6 +2465,7 @@ mt7996_mac_full_reset(struct mt7996_dev *dev) dev->recovery.hw_full_reset = true; + set_bit(MT76_MCU_RESET, &dev->mphy.state); wake_up(&dev->mt76.mcu.wait); ieee80211_stop_queues(hw); @@ -2440,12 +2486,14 @@ mt7996_mac_full_reset(struct mt7996_dev *dev) mt7996_for_each_phy(dev, phy) phy->omac_mask = 0; + dev->mld_idx_mask = 0; + dev->mld_remap_idx_mask = 0; ieee80211_iterate_stations_atomic(hw, mt7996_mac_reset_sta_iter, dev); + mt76_reset_device(&dev->mt76); ieee80211_iterate_active_interfaces_atomic(hw, IEEE80211_IFACE_SKIP_SDATA_NOT_IN_DRIVER, mt7996_mac_reset_vif_iter, dev); - mt76_reset_device(&dev->mt76); INIT_LIST_HEAD(&dev->sta_rc_list); INIT_LIST_HEAD(&dev->twt_list); @@ -2524,8 +2572,8 @@ void mt7996_mac_reset_work(struct work_struct *work) set_bit(MT76_RESET, &dev->mphy.state); set_bit(MT76_MCU_RESET, &dev->mphy.state); - mt76_abort_scan(&dev->mt76); wake_up(&dev->mt76.mcu.wait); + mt76_abort_scan(&dev->mt76); cancel_work_sync(&dev->wed_rro.work); mt7996_for_each_phy(dev, phy) { @@ -2534,6 +2582,8 @@ void mt7996_mac_reset_work(struct work_struct *work) cancel_delayed_work_sync(&phy->mt76->mac_work); } + mutex_lock(&dev->mt76.mutex); + mt76_worker_disable(&dev->mt76.tx_worker); mt76_for_each_q_rx(&dev->mt76, i) { if (mtk_wed_device_active(&dev->mt76.mmio.wed) && @@ -2551,8 +2601,6 @@ void mt7996_mac_reset_work(struct work_struct *work) } napi_disable(&dev->mt76.tx_napi); - mutex_lock(&dev->mt76.mutex); - mt76_wr(dev, MT_MCU_INT_EVENT, MT_MCU_INT_EVENT_DMA_STOPPED); if (mt7996_wait_reset_state(dev, MT_MCU_CMD_RESET_DONE)) { diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/main.c b/drivers/net/wireless/mediatek/mt76/mt7996/main.c index afbcc8c7b18b2f..c6140412ed3227 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/main.c @@ -372,7 +372,8 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif, CONN_STATE_PORT_SECURE, true); rcu_assign_pointer(dev->mt76.wcid[idx], &msta_link->wcid); - ieee80211_iter_keys(mphy->hw, vif, mt7996_key_iter, &it); + if (!mlink->wcid->offchannel) + ieee80211_iter_keys(mphy->hw, vif, mt7996_key_iter, &it); if (!mlink->wcid->offchannel) { if (vif->txq && @@ -1202,15 +1203,9 @@ void mt7996_mac_sta_remove_link(struct mt7996_dev *dev, mt76_wcid_cleanup(&dev->mt76, &msta_link->wcid); if (msta_link->wcid.link_valid) { - struct mt7996_phy *phy; - mt7996_mac_wtbl_update(dev, msta_link->wcid.idx, MT_WTBL_UPDATE_ADM_COUNT_CLEAR); - phy = __mt7996_phy(dev, msta_link->wcid.phy_idx); - if (phy) - phy->mt76->num_sta--; - if (msta->deflink_id == link_id) { msta->deflink_id = IEEE80211_LINK_UNSPECIFIED; if (msta->seclink_id == link_id) { @@ -1236,6 +1231,12 @@ void mt7996_mac_sta_remove_link(struct mt7996_dev *dev, } if (flush) { + struct mt7996_phy *phy = + __mt7996_phy(dev, msta_link->wcid.phy_idx); + + if (phy) + phy->mt76->num_sta--; + rcu_assign_pointer(msta->link[link_id], NULL); rcu_assign_pointer(dev->mt76.wcid[msta_link->wcid.idx], NULL); mt76_wcid_mask_clear(dev->mt76.wcid_mask, msta_link->wcid.idx); @@ -1510,20 +1511,26 @@ static void mt7996_tx(struct ieee80211_hw *hw, struct ieee80211_vif *vif = info->control.vif; struct mt7996_vif *mvif = vif ? (void *)vif->drv_priv : NULL; struct mt76_wcid *wcid = &dev->mt76.global_wcid; + u8 deflink_id = IEEE80211_LINK_UNSPECIFIED; u8 link_id = u32_get_bits(info->control.flags, IEEE80211_TX_CTRL_MLO_LINK); rcu_read_lock(); + if (msta) + deflink_id = msta->deflink_id; + else if (mvif) + deflink_id = mvif->mt76.deflink_id; + + /* the primary link is unset until the first link has been added */ + if (deflink_id >= IEEE80211_MLD_MAX_NUM_LINKS) + deflink_id = 0; + /* Use primary link_id if the value from mac80211 is set to * IEEE80211_LINK_UNSPECIFIED. */ - if (link_id == IEEE80211_LINK_UNSPECIFIED) { - if (msta) - link_id = msta->deflink_id; - else if (mvif) - link_id = mvif->mt76.deflink_id; - } + if (link_id == IEEE80211_LINK_UNSPECIFIED) + link_id = deflink_id; if (vif && ieee80211_vif_is_mld(vif)) { struct ieee80211_bss_conf *link_conf; @@ -1533,7 +1540,7 @@ static void mt7996_tx(struct ieee80211_hw *hw, link_sta = rcu_dereference(sta->link[link_id]); if (!link_sta) - link_sta = rcu_dereference(sta->link[msta->deflink_id]); + link_sta = rcu_dereference(sta->link[deflink_id]); if (link_sta) { memcpy(hdr->addr1, link_sta->addr, ETH_ALEN); @@ -1579,7 +1586,7 @@ static void mt7996_tx(struct ieee80211_hw *hw, if (msta_link) wcid = &msta_link->wcid; } - mt76_tx(mphy, control->sta, wcid, skb); + mt76_tx(mphy, control->sta, mt7996_get_tx_wcid(wcid), skb); unlock: rcu_read_unlock(); } @@ -1883,8 +1890,6 @@ static void mt7996_sta_statistics(struct ieee80211_hw *hw, sinfo->txrate.flags = txrate->flags; sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE); } - sinfo->txrate.flags = txrate->flags; - sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE); sinfo->tx_failed = msta_link->wcid.stats.tx_failed; sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED); @@ -2460,6 +2465,7 @@ mt7996_change_vif_links(struct ieee80211_hw *hw, struct ieee80211_vif *vif, idx = get_free_idx(dev->mld_remap_idx_mask, 0, 15) - 1; if (idx < 0) { + dev->mld_idx_mask &= ~BIT_ULL(mvif->mld_group_idx); ret = -ENOSPC; goto out; } diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c index a1bae5db85007d..645a8b480871e6 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c @@ -2483,7 +2483,7 @@ mt7996_mcu_sta_rate_ctrl_tlv(struct sk_buff *skb, struct mt7996_dev *dev, cap |= STA_CAP_VHT_SGI_160; if (link_sta->vht_cap.cap & IEEE80211_VHT_CAP_TXSTBC) cap |= STA_CAP_VHT_TX_STBC; - if (link_sta->vht_cap.cap & IEEE80211_VHT_CAP_RXSTBC_1) + if (link_sta->vht_cap.cap & IEEE80211_VHT_CAP_RXSTBC_MASK) cap |= STA_CAP_VHT_RX_STBC; if ((vif->type != NL80211_IFTYPE_AP || link_conf->vht_ldpc) && (link_sta->vht_cap.cap & IEEE80211_VHT_CAP_RXLDPC)) @@ -4023,6 +4023,9 @@ int mt7996_mcu_rdd_background_enable(struct mt7996_phy *phy, struct mt7996_dev *dev = phy->dev; int err, region, rdd_idx = mt7996_get_rdd_idx(phy, true); + if (rdd_idx < 0) + return -EINVAL; + if (!chandef) { /* disable offchain */ err = mt7996_mcu_rdd_cmd(dev, RDD_STOP, rdd_idx, 0); if (err) @@ -4445,21 +4448,31 @@ int mt7996_mcu_get_chip_config(struct mt7996_dev *dev, u32 *cap) return ret; /* fixed field */ + if (skb->len < 4) { + dev_kfree_skb(skb); + return -EINVAL; + } skb_pull(skb, 4); buf = skb->data; - while (buf - skb->data < skb->len) { + while (buf - skb->data + sizeof(struct tlv) <= skb->len) { struct tlv *tlv = (struct tlv *)buf; + u16 tlv_len = le16_to_cpu(tlv->len); + + if (tlv_len < sizeof(*tlv) || + tlv_len > skb->len - (buf - skb->data)) + break; switch (le16_to_cpu(tlv->tag)) { case UNI_EVENT_CHIP_CONFIG_EFUSE_VERSION: - *cap = le32_to_cpu(*(__le32 *)(buf + sizeof(*tlv))); + if (tlv_len >= sizeof(*tlv) + sizeof(__le32)) + *cap = le32_to_cpu(*(__le32 *)(buf + sizeof(*tlv))); break; default: break; } - buf += le16_to_cpu(tlv->len); + buf += tlv_len; } dev_kfree_skb(skb); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h index 8902e16508b75e..c673e986ecb52a 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h @@ -917,7 +917,7 @@ enum { #define MT7996_BEACON_UPDATE_SIZE (sizeof(struct bss_req_hdr) + \ sizeof(struct bss_bcn_content_tlv) + \ 4 + MT_TXD_SIZE + \ - sizeof(struct bss_bcn_cntdwn_tlv) + \ + sizeof(struct bss_bcn_cntdwn_tlv) * 2 + \ sizeof(struct bss_bcn_mbss_tlv)) #define MT7996_MAX_BSS_OFFLOAD_SIZE 2048 #define MT7996_MAX_BEACON_SIZE (MT7996_MAX_BSS_OFFLOAD_SIZE - \ diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c b/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c index d9780bb425a719..ac81be5fe02304 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c @@ -17,6 +17,8 @@ static bool wed_enable; module_param(wed_enable, bool, 0644); +#define INVALID_REG_ADDR 0xffffffff + static const struct __base mt7996_reg_base[] = { [WF_AGG_BASE] = { { 0x820e2000, 0x820f2000, 0x830e2000 } }, [WF_ARB_BASE] = { { 0x820e3000, 0x820f3000, 0x830e3000 } }, @@ -54,6 +56,14 @@ static const u32 mt7996_offs[] = { [MIB_BSCR7] = 0x9e8, [MIB_BSCR17] = 0xa10, [MIB_TRDR1] = 0xa28, + [MIB_TSCR0] = 0x6b0, + [MIB_TSCR1] = 0x6b4, + [MIB_TSCR2] = 0x6b8, + [MIB_TSCR3] = 0x6bc, + [MIB_TSCR4] = 0x6c0, + [MIB_TSCR5] = 0x6c4, + [MIB_TSCR6] = 0x6c8, + [MIB_TSCR7] = 0x6d0, [HIF_REMAP_L1] = 0x24, [HIF_REMAP_BASE_L1] = 0x130000, [HIF_REMAP_L2] = 0x1b4, @@ -91,6 +101,14 @@ static const u32 mt7992_offs[] = { [MIB_BSCR7] = 0xae4, [MIB_BSCR17] = 0xb0c, [MIB_TRDR1] = 0xb24, + [MIB_TSCR0] = 0x6b0, + [MIB_TSCR1] = 0x6b4, + [MIB_TSCR2] = 0x6b8, + [MIB_TSCR3] = 0x6bc, + [MIB_TSCR4] = 0x6c0, + [MIB_TSCR5] = 0x6c4, + [MIB_TSCR6] = 0x6c8, + [MIB_TSCR7] = 0x6d0, [HIF_REMAP_L1] = 0x8, [HIF_REMAP_BASE_L1] = 0x40000, [HIF_REMAP_L2] = 0x1b4, @@ -128,6 +146,14 @@ static const u32 mt7990_offs[] = { [MIB_BSCR7] = 0xbd4, [MIB_BSCR17] = 0xbfc, [MIB_TRDR1] = 0xc14, + [MIB_TSCR0] = 0x750, + [MIB_TSCR1] = 0x754, + [MIB_TSCR2] = 0x758, + [MIB_TSCR3] = 0x75c, + [MIB_TSCR4] = 0x760, + [MIB_TSCR5] = 0x764, + [MIB_TSCR6] = 0x768, + [MIB_TSCR7] = 0x770, [HIF_REMAP_L1] = 0x8, [HIF_REMAP_BASE_L1] = 0x40000, [HIF_REMAP_L2] = 0x1b8, @@ -334,7 +360,7 @@ static u32 __mt7996_reg_addr(struct mt7996_dev *dev, u32 addr) return dev->reg.map[i].mapped + ofs; } - return 0; + return INVALID_REG_ADDR; } static u32 __mt7996_reg_remap_addr(struct mt7996_dev *dev, u32 addr) @@ -366,7 +392,7 @@ void mt7996_memcpy_fromio(struct mt7996_dev *dev, void *buf, u32 offset, { u32 addr = __mt7996_reg_addr(dev, offset); - if (addr) { + if (addr != INVALID_REG_ADDR) { memcpy_fromio(buf, dev->mt76.mmio.regs + addr, len); return; } @@ -382,7 +408,7 @@ static u32 mt7996_rr(struct mt76_dev *mdev, u32 offset) struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); u32 addr = __mt7996_reg_addr(dev, offset), val; - if (addr) + if (addr != INVALID_REG_ADDR) return dev->bus_ops->rr(mdev, addr); spin_lock_bh(&dev->reg_lock); @@ -397,7 +423,7 @@ static void mt7996_wr(struct mt76_dev *mdev, u32 offset, u32 val) struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); u32 addr = __mt7996_reg_addr(dev, offset); - if (addr) { + if (addr != INVALID_REG_ADDR) { dev->bus_ops->wr(mdev, addr, val); return; } @@ -412,7 +438,7 @@ static u32 mt7996_rmw(struct mt76_dev *mdev, u32 offset, u32 mask, u32 val) struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); u32 addr = __mt7996_reg_addr(dev, offset); - if (addr) + if (addr != INVALID_REG_ADDR) return dev->bus_ops->rmw(mdev, addr, mask, val); spin_lock_bh(&dev->reg_lock); @@ -464,8 +490,8 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, if (!wed_enable) return 0; - dev->mt76.hwrro_mode = is_mt7996(&dev->mt76) ? MT76_HWRRO_V3 - : MT76_HWRRO_V3_1; + if (hif2 && !mtk_wed_device_active(&dev->mt76.mmio.wed)) + return 0; hif1_ofs = dev->hif2 ? MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0) : 0; @@ -491,23 +517,16 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, wed->wlan.wpdma_tx = wed->wlan.phy_base + hif1_ofs + MT_TXQ_RING_BASE(0) + MT7996_TXQ_BAND2 * MT_RING_SIZE; - if (mt7996_has_hwrro(dev)) { - if (is_mt7996(&dev->mt76)) { - wed->wlan.txfree_tbit = ffs(MT_INT_RX_TXFREE_EXT) - 1; - wed->wlan.wpdma_txfree = wed->wlan.phy_base + hif1_ofs + - MT_RXQ_RING_BASE(0) + - MT7996_RXQ_TXFREE2 * MT_RING_SIZE; - } else { - wed->wlan.txfree_tbit = ffs(MT_INT_RX_TXFREE_BAND1_EXT) - 1; - wed->wlan.wpdma_txfree = wed->wlan.phy_base + hif1_ofs + - MT_RXQ_RING_BASE(0) + - MT7996_RXQ_MCU_WA_EXT * MT_RING_SIZE; - } + if (is_mt7996(&dev->mt76)) { + wed->wlan.txfree_tbit = ffs(MT_INT_RX_TXFREE_EXT) - 1; + wed->wlan.wpdma_txfree = wed->wlan.phy_base + hif1_ofs + + MT_RXQ_RING_BASE(0) + + MT7996_RXQ_TXFREE2 * MT_RING_SIZE; } else { + wed->wlan.txfree_tbit = ffs(MT_INT_RX_TXFREE_BAND1_EXT) - 1; wed->wlan.wpdma_txfree = wed->wlan.phy_base + hif1_ofs + MT_RXQ_RING_BASE(0) + - MT7996_RXQ_MCU_WA_TRI * MT_RING_SIZE; - wed->wlan.txfree_tbit = ffs(MT_INT_RX_DONE_WA_TRI) - 1; + MT7996_RXQ_MCU_WA_EXT * MT_RING_SIZE; } wed->wlan.wpdma_rx_glo = wed->wlan.phy_base + hif1_ofs + MT_WFDMA0_GLO_CFG; @@ -518,7 +537,7 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, wed->wlan.id = MT7996_DEVICE_ID_2; wed->wlan.tx_tbit[0] = ffs(MT_INT_TX_DONE_BAND2) - 1; } else { - wed->wlan.hw_rro = mt7996_has_hwrro(dev); + wed->wlan.hw_rro = true; wed->wlan.wpdma_int = wed->wlan.phy_base + MT_INT_SOURCE_CSR; wed->wlan.wpdma_mask = wed->wlan.phy_base + MT_INT_MASK_CSR; wed->wlan.wpdma_tx = wed->wlan.phy_base + MT_TXQ_RING_BASE(0) + @@ -571,23 +590,15 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, wed->wlan.tx_tbit[0] = ffs(MT_INT_TX_DONE_BAND0) - 1; wed->wlan.tx_tbit[1] = ffs(MT_INT_TX_DONE_BAND1) - 1; if (is_mt7996(&dev->mt76)) { - if (mt7996_has_hwrro(dev)) { - wed->wlan.wpdma_txfree = wed->wlan.phy_base + - MT_RXQ_RING_BASE(0) + - MT7996_RXQ_TXFREE0 * MT_RING_SIZE; - wed->wlan.txfree_tbit = ffs(MT_INT_RX_TXFREE_MAIN) - 1; - } else { - wed->wlan.wpdma_txfree = wed->wlan.phy_base + - MT_RXQ_RING_BASE(0) + - MT7996_RXQ_MCU_WA_MAIN * MT_RING_SIZE; - wed->wlan.txfree_tbit = ffs(MT_INT_RX_DONE_WA_MAIN) - 1; - } + wed->wlan.wpdma_txfree = wed->wlan.phy_base + + MT_RXQ_RING_BASE(0) + + MT7996_RXQ_TXFREE0 * MT_RING_SIZE; + wed->wlan.txfree_tbit = ffs(MT_INT_RX_TXFREE_MAIN) - 1; } else { wed->wlan.txfree_tbit = ffs(MT_INT_RX_DONE_WA_MAIN) - 1; wed->wlan.wpdma_txfree = wed->wlan.phy_base + MT_RXQ_RING_BASE(0) + MT7996_RXQ_MCU_WA_MAIN * MT_RING_SIZE; } - dev->mt76.rx_token_size = MT7996_TOKEN_SIZE + wed->wlan.rx_npkt; if (dev->hif2 && is_mt7992(&dev->mt76)) wed->wlan.id = 0x7992; @@ -610,9 +621,14 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, wed->wlan.reset_complete = mt76_wed_reset_complete; } - if (mtk_wed_device_attach(wed)) { - dev->mt76.hwrro_mode = MT76_HWRRO_OFF; + if (mtk_wed_device_attach(wed)) return 0; + + if (!hif2) { + dev->mt76.hwrro_mode = is_mt7996(&dev->mt76) ? MT76_HWRRO_V3 + : MT76_HWRRO_V3_1; + dev->mt76.rx_token_size = MT7996_TOKEN_SIZE + + wed->wlan.rx_npkt; } *irq = wed->irq; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h index 0d6488522ba711..2e3edac2b5719e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h @@ -871,6 +871,7 @@ bool mt7996_mac_wtbl_update(struct mt7996_dev *dev, int idx, u32 mask); void mt7996_mac_reset_counters(struct mt7996_phy *phy); void mt7996_mac_cca_stats_reset(struct mt7996_phy *phy); void mt7996_mac_enable_nf(struct mt7996_dev *dev, u8 band); +struct mt76_wcid *mt7996_get_tx_wcid(struct mt76_wcid *wcid); void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi, struct sk_buff *skb, struct mt76_wcid *wcid, struct ieee80211_key_conf *key, int pid, diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/regs.h b/drivers/net/wireless/mediatek/mt76/mt7996/regs.h index c6379933b6c363..8ff78cf6eb042a 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/regs.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/regs.h @@ -64,6 +64,14 @@ enum offs_rev { MIB_BSCR7, MIB_BSCR17, MIB_TRDR1, + MIB_TSCR0, + MIB_TSCR1, + MIB_TSCR2, + MIB_TSCR3, + MIB_TSCR4, + MIB_TSCR5, + MIB_TSCR6, + MIB_TSCR7, HIF_REMAP_L1, HIF_REMAP_BASE_L1, HIF_REMAP_L2, @@ -250,9 +258,9 @@ enum offs_rev { #define MT_MIB_BSCR7(_band) MT_WF_MIB(_band, __OFFS(MIB_BSCR7)) #define MT_MIB_BSCR17(_band) MT_WF_MIB(_band, __OFFS(MIB_BSCR17)) -#define MT_MIB_TSCR5(_band) MT_WF_MIB(_band, 0x6c4) -#define MT_MIB_TSCR6(_band) MT_WF_MIB(_band, 0x6c8) -#define MT_MIB_TSCR7(_band) MT_WF_MIB(_band, 0x6d0) +#define MT_MIB_TSCR5(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR5)) +#define MT_MIB_TSCR6(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR6)) +#define MT_MIB_TSCR7(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR7)) #define MT_MIB_RSCR1(_band) MT_WF_MIB(_band, __OFFS(MIB_RSCR1)) /* rx mpdu counter, full 32 bits */ @@ -268,14 +276,14 @@ enum offs_rev { #define MT_MIB_RSCR36(_band) MT_WF_MIB(_band, __OFFS(MIB_RSCR36)) /* tx ampdu cnt, full 32 bits */ -#define MT_MIB_TSCR0(_band) MT_WF_MIB(_band, 0x6b0) -#define MT_MIB_TSCR2(_band) MT_WF_MIB(_band, 0x6b8) +#define MT_MIB_TSCR0(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR0)) +#define MT_MIB_TSCR2(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR2)) /* counts all mpdus in ampdu, regardless of success */ -#define MT_MIB_TSCR3(_band) MT_WF_MIB(_band, 0x6bc) +#define MT_MIB_TSCR3(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR3)) /* counts all successfully tx'd mpdus in ampdu */ -#define MT_MIB_TSCR4(_band) MT_WF_MIB(_band, 0x6c0) +#define MT_MIB_TSCR4(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR4)) /* rx ampdu count, 32-bit */ #define MT_MIB_RSCR27(_band) MT_WF_MIB(_band, __OFFS(MIB_RSCR27)) @@ -299,7 +307,7 @@ enum offs_rev { #define MT_MIB_RVSR1(_band) MT_WF_MIB(_band, __OFFS(MIB_RVSR1)) /* rx blockack count, 32 bits */ -#define MT_MIB_TSCR1(_band) MT_WF_MIB(_band, 0x6b4) +#define MT_MIB_TSCR1(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR1)) #define MT_MIB_BTSCR0(_band) MT_WF_MIB(_band, 0x5e0) #define MT_MIB_BTSCR5(_band) MT_WF_MIB(_band, __OFFS(MIB_BTSCR5)) diff --git a/drivers/net/wireless/mediatek/mt76/scan.c b/drivers/net/wireless/mediatek/mt76/scan.c index 7fe1b1fbb699b7..3cb11689d4bf25 100644 --- a/drivers/net/wireless/mediatek/mt76/scan.c +++ b/drivers/net/wireless/mediatek/mt76/scan.c @@ -16,10 +16,17 @@ static void mt76_scan_complete(struct mt76_dev *dev, bool abort) clear_bit(MT76_SCANNING, &phy->state); - if (dev->scan.chan && phy->main_chandef.chan && phy->offchannel && + /* Re-program the operating channel even when the scan never left it: + * any channel set during the scan ran with MT76_SCANNING held, which + * left DFS radar detection disabled + */ + if (phy->main_chandef.chan && !test_bit(MT76_MCU_RESET, &dev->phy.state)) { + bool offchannel = phy->offchannel; + mt76_set_channel(phy, &phy->main_chandef, false); - mt76_offchannel_notify(phy, false); + if (offchannel) + mt76_offchannel_notify(phy, false); } mt76_put_vif_phy_link(phy, dev->scan.vif, dev->scan.mlink); memset(&dev->scan, 0, sizeof(dev->scan)); @@ -48,6 +55,7 @@ mt76_scan_send_probe(struct mt76_dev *dev, struct cfg80211_ssid *ssid) struct mt76_phy *phy = dev->scan.phy; struct ieee80211_tx_info *info; struct sk_buff *skb; + u8 link_id; skb = ieee80211_probereq_get(phy->hw, vif->addr, ssid->ssid, ssid->ssid_len, req->ie_len); @@ -77,6 +85,10 @@ mt76_scan_send_probe(struct mt76_dev *dev, struct cfg80211_ssid *ssid) info->flags |= IEEE80211_TX_CTL_NO_CCK_RATE; info->control.flags |= IEEE80211_TX_CTRL_DONT_USE_RATE_MASK; + link_id = mvif->wcid ? mvif->wcid->link_id : IEEE80211_LINK_UNSPECIFIED; + info->control.flags &= ~IEEE80211_TX_CTRL_MLO_LINK; + info->control.flags |= u32_encode_bits(link_id, IEEE80211_TX_CTRL_MLO_LINK); + mt76_tx(phy, NULL, mvif->wcid, skb); out: @@ -206,6 +218,7 @@ int mt76_hw_scan(struct ieee80211_hw *hw, struct ieee80211_vif *vif, dev->scan.vif = vif; dev->scan.phy = phy; dev->scan.mlink = mlink; + set_bit(MT76_SCANNING, &phy->state); ieee80211_queue_delayed_work(dev->phy.hw, &dev->scan_work, 0); out: diff --git a/drivers/net/wireless/mediatek/mt76/sdio.c b/drivers/net/wireless/mediatek/mt76/sdio.c index 8bae77c761beae..ba5f123f7e39dc 100644 --- a/drivers/net/wireless/mediatek/mt76/sdio.c +++ b/drivers/net/wireless/mediatek/mt76/sdio.c @@ -519,6 +519,10 @@ mt76s_tx_queue_skb(struct mt76_phy *phy, struct mt76_queue *q, enum mt76_txq_id qid, struct sk_buff *skb, struct mt76_wcid *wcid, struct ieee80211_sta *sta) { + struct ieee80211_tx_status status = { + .sta = sta, + }; + struct mt76_tx_info tx_info = { .skb = skb, }; @@ -531,8 +535,13 @@ mt76s_tx_queue_skb(struct mt76_phy *phy, struct mt76_queue *q, skb->prev = skb->next = NULL; err = dev->drv->tx_prepare_skb(dev, NULL, qid, wcid, sta, &tx_info); - if (err < 0) + if (err < 0) { + status.skb = tx_info.skb; + spin_lock_bh(&dev->rx_lock); + ieee80211_tx_status_ext(dev->hw, &status); + spin_unlock_bh(&dev->rx_lock); return err; + } q->entry[q->head].skb = tx_info.skb; q->entry[q->head].buf_sz = len; diff --git a/drivers/net/wireless/mediatek/mt76/tx.c b/drivers/net/wireless/mediatek/mt76/tx.c index f96d9c47185354..b03be0eb471282 100644 --- a/drivers/net/wireless/mediatek/mt76/tx.c +++ b/drivers/net/wireless/mediatek/mt76/tx.c @@ -324,10 +324,6 @@ __mt76_tx_queue_skb(struct mt76_phy *phy, int qid, struct sk_buff *skb, if (idx < 0 || !sta) return idx; - wcid = (struct mt76_wcid *)sta->drv_priv; - if (!wcid->sta) - return idx; - q->entry[idx].wcid = wcid->idx; if (!non_aql) @@ -635,6 +631,7 @@ mt76_txq_schedule_pending_wcid(struct mt76_phy *phy, struct mt76_wcid *wcid, !ieee80211_is_data_present(hdr->frame_control) && (!ieee80211_is_bufferable_mmpdu(skb) || ieee80211_is_deauth(hdr->frame_control) || + ieee80211_is_disassoc(hdr->frame_control) || head == &wcid->tx_offchannel)) qid = MT_TXQ_PSD; @@ -686,8 +683,8 @@ void mt76_txq_schedule_pending(struct mt76_phy *phy) ret = mt76_txq_schedule_pending_wcid(phy, wcid, &wcid->tx_pending); spin_lock(&phy->tx_lock); - if (!skb_queue_empty(&wcid->tx_pending) && - !skb_queue_empty(&wcid->tx_offchannel) && + if ((!skb_queue_empty(&wcid->tx_pending) || + !skb_queue_empty(&wcid->tx_offchannel)) && list_empty(&wcid->tx_list)) list_add_tail(&wcid->tx_list, &phy->tx_list); } diff --git a/drivers/net/wireless/mediatek/mt76/usb.c b/drivers/net/wireless/mediatek/mt76/usb.c index d9638a9b749b66..345f1d9c1947cb 100644 --- a/drivers/net/wireless/mediatek/mt76/usb.c +++ b/drivers/net/wireless/mediatek/mt76/usb.c @@ -30,6 +30,8 @@ int __mt76u_vendor_request(struct mt76_dev *dev, u8 req, u8 req_type, for (i = 0; i < MT_VEND_REQ_MAX_RETRY; i++) { if (test_bit(MT76_REMOVED, &dev->phy.state)) return -EIO; + if (dev->usb.ctrl_timeout && atomic_read(&dev->bus_hung)) + return -EIO; ret = usb_control_msg(udev, pipe, req, req_type, val, offset, buf, len, MT_VEND_REQ_TOUT_MS); @@ -42,6 +44,15 @@ int __mt76u_vendor_request(struct mt76_dev *dev, u8 req, u8 req_type, dev_err(dev->dev, "vendor request req:%02x off:%04x failed:%d\n", req, offset, ret); + + if (dev->usb.ctrl_timeout) { + atomic_set(&dev->bus_hung, true); + dev_err(dev->dev, "vendor request req:%02x off:%04x timed out, marking bus hung\n", + req, offset); + dev->usb.ctrl_timeout(dev, ret); + return ret; + } + return ret; } EXPORT_SYMBOL_GPL(__mt76u_vendor_request); diff --git a/drivers/net/wireless/realtek/rtlwifi/pci.c b/drivers/net/wireless/realtek/rtlwifi/pci.c index 73018a0498b4b9..de74ff8f6eee74 100644 --- a/drivers/net/wireless/realtek/rtlwifi/pci.c +++ b/drivers/net/wireless/realtek/rtlwifi/pci.c @@ -2227,13 +2227,17 @@ int rtl_pci_probe(struct pci_dev *pdev, rtl_dbg(rtlpriv, COMP_INIT, DBG_DMESG, "%s: failed to register IRQ handler\n", wiphy_name(hw->wiphy)); - goto fail3; + goto fail6; } rtlpci->irq_alloc = 1; set_bit(RTL_STATUS_INTERFACE_START, &rtlpriv->status); return 0; +fail6: + rtl_deinit_rfkill(hw); + rtl_debug_remove_one(hw); + ieee80211_unregister_hw(hw); fail5: rtl_pci_deinit(hw); fail4: diff --git a/drivers/net/wireless/realtek/rtw89/debug.c b/drivers/net/wireless/realtek/rtw89/debug.c index 8f5af873e09f5e..5786120602ab0b 100644 --- a/drivers/net/wireless/realtek/rtw89/debug.c +++ b/drivers/net/wireless/realtek/rtw89/debug.c @@ -4348,7 +4348,7 @@ static const char *rtw89_ppdu_str(struct rtw89_dev *rtwdev, u8 type, u8 subtype) const struct rtw89_chip_info *chip = rtwdev->chip; const struct rtw89_ppdu_info *ppdu_info; - if (type > ARRAY_SIZE(rtw89_ppdu_infos)) + if (type >= ARRAY_SIZE(rtw89_ppdu_infos)) return "RSVD"; ppdu_info = &rtw89_ppdu_infos[type]; diff --git a/drivers/net/wireless/realtek/rtw89/fw.c b/drivers/net/wireless/realtek/rtw89/fw.c index d6a594b75ab202..5edad2d25ae4b3 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.c +++ b/drivers/net/wireless/realtek/rtw89/fw.c @@ -3439,7 +3439,7 @@ int rtw89_fw_h2c_lps_ml_cmn_info_v1(struct rtw89_dev *rtwdev, h2c->rfe_type = efuse->rfe_type; h2c->rssi_main = U8_MAX; - memset(h2c->link_id, 0xfe, RTW89_BB_PS_LINK_BUF_MAX); + memset(h2c->link_id, RTW89_BB_PS_LINK_ID_SKIP, RTW89_BB_PS_LINK_BUF_MAX); rtw89_vif_for_each_link(rtwvif, rtwvif_link, link_id) { u8 phy_idx = rtwvif_link->phy_idx; @@ -3447,7 +3447,7 @@ int rtw89_fw_h2c_lps_ml_cmn_info_v1(struct rtw89_dev *rtwdev, bb = rtw89_get_bb_ctx(rtwdev, phy_idx); chan = rtw89_chan_get(rtwdev, rtwvif_link->chanctx_idx); - h2c->link_id[phy_idx] = phy_idx; + h2c->link_id[phy_idx] = link_id; h2c->central_ch[phy_idx] = chan->channel; h2c->pri_ch[phy_idx] = chan->primary_channel; h2c->band[phy_idx] = chan->band_type; @@ -3729,7 +3729,7 @@ int rtw89_fw_h2c_default_cmac_tbl_be(struct rtw89_dev *rtwdev, le32_encode_bits(4, BE_CCTL_INFO_W1_RTS_RTY_LOWEST_RATE); h2c->m1 = cpu_to_le32(BE_CCTL_INFO_W1_ALL); - h2c->w1 = le32_encode_bits(preld, BE_CCTL_INFO_W2_PRELOAD_ENABLE); + h2c->w2 = le32_encode_bits(preld, BE_CCTL_INFO_W2_PRELOAD_ENABLE); h2c->m2 = cpu_to_le32(BE_CCTL_INFO_W2_ALL); h2c->m3 = cpu_to_le32(BE_CCTL_INFO_W3_ALL); @@ -11780,7 +11780,7 @@ static void rtw89_fw_cmd_ofld_write_rf(struct rtw89_dev *rtwdev, static void rtw89_fw_cmd_ofld_udelay(struct rtw89_dev *rtwdev, u32 us) { struct rtw89_fw_cmd_ofld_arg cmd = { - .src = RTW89_FW_CMD_OFLD_SRC_OTHER, + .src = RTW89_FW_CMD_OFLD_SRC_MAC, .type = RTW89_FW_CMD_OFLD_DELAY, .value = us, }; @@ -11794,7 +11794,7 @@ static void rtw89_fw_cmd_ofld_udelay(struct rtw89_dev *rtwdev, u32 us) static void rtw89_fw_cmd_ofld_mdelay(struct rtw89_dev *rtwdev, u32 ms) { struct rtw89_fw_cmd_ofld_arg cmd = { - .src = RTW89_FW_CMD_OFLD_SRC_OTHER, + .src = RTW89_FW_CMD_OFLD_SRC_MAC, .type = RTW89_FW_CMD_OFLD_DELAY, .value = ms * 1000, }; diff --git a/drivers/net/wireless/realtek/rtw89/fw.h b/drivers/net/wireless/realtek/rtw89/fw.h index 20721d5209aa3a..8c90865dfa3b54 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.h +++ b/drivers/net/wireless/realtek/rtw89/fw.h @@ -2053,6 +2053,8 @@ enum rtw89_bb_link_rx_gain_table_type { RTW89_BB_PS_LINK_RX_GAIN_TAB_MAX, }; +#define RTW89_BB_PS_LINK_ID_SKIP 0xfe + enum rtw89_bb_ps_link_buf_id { RTW89_BB_PS_LINK_BUF_0 = 0x00, RTW89_BB_PS_LINK_BUF_1 = 0x01, @@ -3142,7 +3144,6 @@ enum rtw89_fw_cmd_ofld_arg_src { RTW89_FW_CMD_OFLD_SRC_RF, RTW89_FW_CMD_OFLD_SRC_MAC, RTW89_FW_CMD_OFLD_SRC_RF_DDIE, - RTW89_FW_CMD_OFLD_SRC_OTHER, }; enum rtw89_fw_cmd_ofld_arg_type { diff --git a/drivers/net/wireless/realtek/rtw89/mac.c b/drivers/net/wireless/realtek/rtw89/mac.c index 8c395517bd2fe1..99de1b20297685 100644 --- a/drivers/net/wireless/realtek/rtw89/mac.c +++ b/drivers/net/wireless/realtek/rtw89/mac.c @@ -5167,7 +5167,7 @@ static void rtw89_mac_check_he_obss_narrow_bw_ru_iter(struct wiphy *wiphy, elem = cfg80211_find_elem(WLAN_EID_EXT_CAPABILITY, ies->data, ies->len); - if (!elem || elem->datalen < 10 || + if (!elem || elem->datalen < 11 || !(elem->data[10] & WLAN_EXT_CAPA10_OBSS_NARROW_BW_RU_TOLERANCE_SUPPORT)) *tolerated = false; rcu_read_unlock(); diff --git a/drivers/net/wireless/realtek/rtw89/mac80211.c b/drivers/net/wireless/realtek/rtw89/mac80211.c index 9baedfde708542..c1be69a3c19264 100644 --- a/drivers/net/wireless/realtek/rtw89/mac80211.c +++ b/drivers/net/wireless/realtek/rtw89/mac80211.c @@ -826,11 +826,36 @@ static int rtw89_ops_start_ap(struct ieee80211_hw *hw, ether_addr_copy(rtwvif_link->bssid, link_conf->bssid); rtw89_cam_bssid_changed(rtwdev, rtwvif_link); - rtw89_mac_port_update(rtwdev, rtwvif_link); - rtw89_chip_h2c_assoc_cmac_tbl(rtwdev, rtwvif_link, NULL); - rtw89_fw_h2c_role_maintain(rtwdev, rtwvif_link, NULL, RTW89_ROLE_TYPE_CHANGE); - rtw89_fw_h2c_join_info(rtwdev, rtwvif_link, NULL, true); - rtw89_fw_h2c_cam(rtwdev, rtwvif_link, NULL, NULL, RTW89_ROLE_TYPE_CHANGE); + ret = rtw89_mac_port_update(rtwdev, rtwvif_link); + if (ret) { + rtw89_warn(rtwdev, "failed to update mac port\n"); + return ret; + } + + ret = rtw89_chip_h2c_assoc_cmac_tbl(rtwdev, rtwvif_link, NULL); + if (ret) { + rtw89_warn(rtwdev, "failed to send h2c cmac table\n"); + return ret; + } + + ret = rtw89_fw_h2c_role_maintain(rtwdev, rtwvif_link, NULL, RTW89_ROLE_TYPE_CHANGE); + if (ret) { + rtw89_warn(rtwdev, "failed to send h2c role info\n"); + return ret; + } + + ret = rtw89_fw_h2c_join_info(rtwdev, rtwvif_link, NULL, true); + if (ret) { + rtw89_warn(rtwdev, "failed to send h2c join info\n"); + return ret; + } + + ret = rtw89_fw_h2c_cam(rtwdev, rtwvif_link, NULL, NULL, RTW89_ROLE_TYPE_CHANGE); + if (ret) { + rtw89_warn(rtwdev, "failed to send h2c cam\n"); + return ret; + } + rtw89_chip_rfk_channel(rtwdev, rtwvif_link); if (RTW89_CHK_FW_FEATURE(NOTIFY_AP_INFO, &rtwdev->fw)) { diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852a.c b/drivers/net/wireless/realtek/rtw89/rtw8852a.c index 2c1f166e687f06..e27a81bd8b77d6 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852a.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852a.c @@ -2212,7 +2212,7 @@ static void rtw8852a_query_ppdu(struct rtw89_dev *rtwdev, u8 raw; if (!status->signal) { - if (phy_ppdu->to_self) + if (phy_ppdu->to_self && ewma_rssi_read(&bb->bcn_rssi)) raw = ewma_rssi_read(&bb->bcn_rssi); else raw = max(rx_power[RF_PATH_A], rx_power[RF_PATH_B]); diff --git a/drivers/net/wireless/virtual/mac80211_hwsim_main.c b/drivers/net/wireless/virtual/mac80211_hwsim_main.c index 75caa97becc8dd..b4cabbfa9cdbac 100644 --- a/drivers/net/wireless/virtual/mac80211_hwsim_main.c +++ b/drivers/net/wireless/virtual/mac80211_hwsim_main.c @@ -2314,6 +2314,7 @@ static int mac80211_hwsim_start(struct ieee80211_hw *hw) static void mac80211_hwsim_stop(struct ieee80211_hw *hw, bool suspend) { struct mac80211_hwsim_data *data = hw->priv; + struct sk_buff *skb; int i; data->started = false; @@ -2321,8 +2322,8 @@ static void mac80211_hwsim_stop(struct ieee80211_hw *hw, bool suspend) for (i = 0; i < ARRAY_SIZE(data->link_data); i++) hrtimer_cancel(&data->link_data[i].beacon_timer); - while (!skb_queue_empty(&data->pending)) - ieee80211_free_txskb(hw, skb_dequeue(&data->pending)); + while ((skb = skb_dequeue(&data->pending))) + ieee80211_free_txskb(hw, skb); wiphy_dbg(hw->wiphy, "%s\n", __func__); } diff --git a/drivers/net/wireless/zydas/zd1211rw/zd_usb.c b/drivers/net/wireless/zydas/zd1211rw/zd_usb.c index 966d8ccb0dbcd6..98102c663434bf 100644 --- a/drivers/net/wireless/zydas/zd1211rw/zd_usb.c +++ b/drivers/net/wireless/zydas/zd1211rw/zd_usb.c @@ -1353,6 +1353,14 @@ static int probe(struct usb_interface *intf, const struct usb_device_id *id) struct zd_usb *usb; struct ieee80211_hw *hw = NULL; + /* + * ZD1211 devices are single-function. Reject secondary interfaces + * to prevent multiple instances from conflicting on hardcoded endpoints + * and triggering recursive locking warnings. + */ + if (intf->cur_altsetting->desc.bInterfaceNumber != 0) + return -ENODEV; + print_id(udev); if (id->driver_info & DEVICE_INSTALLER) diff --git a/drivers/nfc/pn533/pn533.c b/drivers/nfc/pn533/pn533.c index 6db9ec90f594b6..76081a99b4501f 100644 --- a/drivers/nfc/pn533/pn533.c +++ b/drivers/nfc/pn533/pn533.c @@ -434,6 +434,18 @@ done: return rc; } +static int pn533_send_cmd_frame(struct pn533 *dev, struct pn533_cmd *cmd) +{ + struct sk_buff *req = cmd->req; + int rc; + + skb_get(req); + dev->cmd = cmd; + rc = dev->phy_ops->send_frame(dev, req); + dev_kfree_skb(req); + return rc; +} + static int __pn533_send_async(struct pn533 *dev, u8 cmd_code, struct sk_buff *req, pn533_send_async_complete_t complete_cb, @@ -458,8 +470,7 @@ static int __pn533_send_async(struct pn533 *dev, u8 cmd_code, mutex_lock(&dev->cmd_lock); if (!dev->cmd_pending) { - dev->cmd = cmd; - rc = dev->phy_ops->send_frame(dev, req); + rc = pn533_send_cmd_frame(dev, cmd); if (rc) { dev->cmd = NULL; goto error; @@ -529,8 +540,7 @@ static int pn533_send_cmd_direct_async(struct pn533 *dev, u8 cmd_code, pn533_build_cmd_frame(dev, cmd_code, req); - dev->cmd = cmd; - rc = dev->phy_ops->send_frame(dev, req); + rc = pn533_send_cmd_frame(dev, cmd); if (rc < 0) { dev->cmd = NULL; kfree(cmd); @@ -569,8 +579,7 @@ static void pn533_wq_cmd(struct work_struct *work) mutex_unlock(&dev->cmd_lock); - dev->cmd = cmd; - rc = dev->phy_ops->send_frame(dev, cmd->req); + rc = pn533_send_cmd_frame(dev, cmd); if (rc < 0) { dev->cmd = NULL; dev_kfree_skb(cmd->req); diff --git a/drivers/nvdimm/label.c b/drivers/nvdimm/label.c index ec12ce72cfe2ca..dea2eee86d132e 100644 --- a/drivers/nvdimm/label.c +++ b/drivers/nvdimm/label.c @@ -145,10 +145,21 @@ static int __nd_label_validate(struct nvdimm_drvdata *ndd) /* label sizes larger than 128 arrived with v1.2 */ version = __le16_to_cpu(nsindex[i]->major) * 100 + __le16_to_cpu(nsindex[i]->minor); - if (version >= 102) + if (version >= 102) { + /* + * labelsize feeds the shift below; only 0 (128-byte) + * and 1 (256-byte) are valid -- a larger value would + * overflow or exceed the width of int. + */ + if (nsindex[i]->labelsize > 1) { + dev_dbg(dev, "nsindex%d labelsize: %d invalid\n", + i, nsindex[i]->labelsize); + continue; + } labelsize = 1 << (7 + nsindex[i]->labelsize); - else + } else { labelsize = 128; + } if (labelsize != sizeof_namespace_label(ndd)) { dev_dbg(dev, "nsindex%d labelsize %d invalid\n", diff --git a/drivers/nvdimm/nd_virtio.c b/drivers/nvdimm/nd_virtio.c index 4176046627beb3..cbc0f53398f090 100644 --- a/drivers/nvdimm/nd_virtio.c +++ b/drivers/nvdimm/nd_virtio.c @@ -9,26 +9,48 @@ #include "virtio_pmem.h" #include "nd.h" +struct virtio_pmem_flush_work { + struct work_struct work; + struct nd_region *nd_region; + struct bio *bio; +}; + +static void virtio_pmem_req_release(struct kref *kref) +{ + struct virtio_pmem_request *req; + + req = container_of(kref, struct virtio_pmem_request, kref); + kfree(req); +} + +static void virtio_pmem_wake_one_waiter(struct virtio_pmem *vpmem) +{ + struct virtio_pmem_request *req_buf; + + if (list_empty(&vpmem->req_list)) + return; + + req_buf = list_first_entry(&vpmem->req_list, + struct virtio_pmem_request, list); + list_del_init(&req_buf->list); + WRITE_ONCE(req_buf->wq_buf_avail, true); + wake_up(&req_buf->wq_buf); +} + /* The interrupt handler */ void virtio_pmem_host_ack(struct virtqueue *vq) { struct virtio_pmem *vpmem = vq->vdev->priv; - struct virtio_pmem_request *req_data, *req_buf; + struct virtio_pmem_request *req_data; unsigned long flags; unsigned int len; spin_lock_irqsave(&vpmem->pmem_lock, flags); while ((req_data = virtqueue_get_buf(vq, &len)) != NULL) { - req_data->done = true; + virtio_pmem_wake_one_waiter(vpmem); + WRITE_ONCE(req_data->done, true); wake_up(&req_data->host_acked); - - if (!list_empty(&vpmem->req_list)) { - req_buf = list_first_entry(&vpmem->req_list, - struct virtio_pmem_request, list); - req_buf->wq_buf_avail = true; - wake_up(&req_buf->wq_buf); - list_del(&req_buf->list); - } + kref_put(&req_data->kref, virtio_pmem_req_release); } spin_unlock_irqrestore(&vpmem->pmem_lock, flags); } @@ -59,7 +81,8 @@ static int virtio_pmem_flush(struct nd_region *nd_region) if (!req_data) return -ENOMEM; - req_data->done = false; + kref_init(&req_data->kref); + WRITE_ONCE(req_data->done, false); init_waitqueue_head(&req_data->host_acked); init_waitqueue_head(&req_data->wq_buf); INIT_LIST_HEAD(&req_data->list); @@ -76,18 +99,32 @@ static int virtio_pmem_flush(struct nd_region *nd_region) * to req_list and wait for host_ack to wake us up when free * slots are available. */ - while ((err = virtqueue_add_sgs(vpmem->req_vq, sgs, 1, 1, req_data, - GFP_ATOMIC)) == -ENOSPC) { + for (;;) { + err = virtqueue_add_sgs(vpmem->req_vq, sgs, 1, 1, req_data, + GFP_ATOMIC); + if (!err) { + /* + * Take the virtqueue reference while @pmem_lock is + * held so completion cannot run concurrently. + */ + kref_get(&req_data->kref); + break; + } - dev_info(&vdev->dev, "failed to send command to virtio pmem device, no free slots in the virtqueue\n"); - req_data->wq_buf_avail = false; + if (err != -ENOSPC) + break; + + dev_info_ratelimited(&vdev->dev, + "failed to send command to virtio pmem device, no free slots in the virtqueue\n"); + WRITE_ONCE(req_data->wq_buf_avail, false); list_add_tail(&req_data->list, &vpmem->req_list); spin_unlock_irqrestore(&vpmem->pmem_lock, flags); /* A host response results in "host_ack" getting called */ - wait_event(req_data->wq_buf, req_data->wq_buf_avail); + wait_event(req_data->wq_buf, READ_ONCE(req_data->wq_buf_avail)); spin_lock_irqsave(&vpmem->pmem_lock, flags); } + err1 = virtqueue_kick(vpmem->req_vq); spin_unlock_irqrestore(&vpmem->pmem_lock, flags); /* @@ -99,38 +136,54 @@ static int virtio_pmem_flush(struct nd_region *nd_region) err = -EIO; } else { /* A host response results in "host_ack" getting called */ - wait_event(req_data->host_acked, req_data->done); + wait_event(req_data->host_acked, READ_ONCE(req_data->done)); err = le32_to_cpu(req_data->resp.ret); } - kfree(req_data); + kref_put(&req_data->kref, virtio_pmem_req_release); return err; }; +static void virtio_pmem_flush_work(struct work_struct *work) +{ + struct virtio_pmem_flush_work *flush; + int err; + + flush = container_of(work, struct virtio_pmem_flush_work, work); + err = virtio_pmem_flush(flush->nd_region); + if (err > 0) + err = -EIO; + if (err) + flush->bio->bi_status = errno_to_blk_status(err); + bio_endio(flush->bio); + kfree(flush); +} + /* The asynchronous flush callback function */ int async_pmem_flush(struct nd_region *nd_region, struct bio *bio) { - /* - * Create child bio for asynchronous flush and chain with - * parent bio. Otherwise directly call nd_region flush. - */ - if (bio && bio->bi_iter.bi_sector != -1) { - struct bio *child = bio_alloc(bio->bi_bdev, 0, - REQ_OP_WRITE | REQ_PREFLUSH, - GFP_ATOMIC); + struct virtio_device *vdev = nd_region->provider_data; + struct virtio_pmem *vpmem = vdev->priv; + struct virtio_pmem_flush_work *flush; + int err; - if (!child) + if (bio && bio->bi_iter.bi_sector != -1) { + flush = kmalloc_obj(*flush, GFP_NOIO); + if (!flush) return -ENOMEM; - bio_clone_blkg_association(child, bio); - child->bi_iter.bi_sector = -1; - bio_chain(child, bio); - submit_bio(child); - return 0; + + INIT_WORK(&flush->work, virtio_pmem_flush_work); + flush->nd_region = nd_region; + flush->bio = bio; + queue_work(vpmem->flush_wq, &flush->work); + return NVDIMM_FLUSH_ASYNC; } - if (virtio_pmem_flush(nd_region)) + + err = virtio_pmem_flush(nd_region); + if (err > 0) return -EIO; - return 0; + return err; }; EXPORT_SYMBOL_GPL(async_pmem_flush); MODULE_DESCRIPTION("Virtio Persistent Memory Driver"); diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 92c67fbbc1c85d..ff27e475921641 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -208,8 +208,14 @@ static void pmem_submit_bio(struct bio *bio) struct pmem_device *pmem = bio->bi_bdev->bd_disk->private_data; struct nd_region *nd_region = to_region(pmem); - if (bio->bi_opf & REQ_PREFLUSH) - ret = nvdimm_flush(nd_region, bio); + if (bio->bi_opf & REQ_PREFLUSH) { + ret = nvdimm_flush(nd_region, NULL); + if (ret) { + bio->bi_status = errno_to_blk_status(ret); + bio_endio(bio); + return; + } + } do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue); if (do_acct) @@ -229,8 +235,11 @@ static void pmem_submit_bio(struct bio *bio) if (do_acct) bio_end_io_acct(bio, start); - if (bio->bi_opf & REQ_FUA) + if ((bio->bi_opf & REQ_FUA) && !bio->bi_status) { ret = nvdimm_flush(nd_region, bio); + if (ret == NVDIMM_FLUSH_ASYNC) + return; + } if (ret) bio->bi_status = errno_to_blk_status(ret); diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c index 5e079d61cbaa32..24f42b4650ba66 100644 --- a/drivers/nvdimm/region_devs.c +++ b/drivers/nvdimm/region_devs.c @@ -1093,7 +1093,10 @@ int nvdimm_flush(struct nd_region *nd_region, struct bio *bio) if (!nd_region->flush) rc = generic_nvdimm_flush(nd_region); else { - if (nd_region->flush(nd_region, bio)) + rc = nd_region->flush(nd_region, bio); + if (rc > 0) + return rc; + if (rc && rc != -ENOMEM) rc = -EIO; } diff --git a/drivers/nvdimm/virtio_pmem.c b/drivers/nvdimm/virtio_pmem.c index 77b1966619059c..9cf822a6c0c38e 100644 --- a/drivers/nvdimm/virtio_pmem.c +++ b/drivers/nvdimm/virtio_pmem.c @@ -67,10 +67,17 @@ static int virtio_pmem_probe(struct virtio_device *vdev) mutex_init(&vpmem->flush_lock); vpmem->vdev = vdev; vdev->priv = vpmem; + vpmem->flush_wq = alloc_ordered_workqueue("virtio-pmem-flush", + WQ_MEM_RECLAIM); + if (!vpmem->flush_wq) { + err = -ENOMEM; + goto out_err; + } + err = init_vq(vpmem); if (err) { dev_err(&vdev->dev, "failed to initialize virtio pmem vq's\n"); - goto out_err; + goto out_wq; } if (virtio_has_feature(vdev, VIRTIO_PMEM_F_SHMEM_REGION)) { @@ -131,6 +138,8 @@ out_nd: nvdimm_bus_unregister(vpmem->nvdimm_bus); out_vq: vdev->config->del_vqs(vdev); +out_wq: + destroy_workqueue(vpmem->flush_wq); out_err: return err; } @@ -138,14 +147,20 @@ out_err: static void virtio_pmem_remove(struct virtio_device *vdev) { struct nvdimm_bus *nvdimm_bus = dev_get_drvdata(&vdev->dev); + struct virtio_pmem *vpmem = vdev->priv; nvdimm_bus_unregister(nvdimm_bus); + drain_workqueue(vpmem->flush_wq); vdev->config->del_vqs(vdev); virtio_reset_device(vdev); + destroy_workqueue(vpmem->flush_wq); } static int virtio_pmem_freeze(struct virtio_device *vdev) { + struct virtio_pmem *vpmem = vdev->priv; + + drain_workqueue(vpmem->flush_wq); vdev->config->del_vqs(vdev); virtio_reset_device(vdev); diff --git a/drivers/nvdimm/virtio_pmem.h b/drivers/nvdimm/virtio_pmem.h index f72cf17f9518fb..3af92588bd9d18 100644 --- a/drivers/nvdimm/virtio_pmem.h +++ b/drivers/nvdimm/virtio_pmem.h @@ -12,11 +12,14 @@ #include #include +#include #include #include #include +#include struct virtio_pmem_request { + struct kref kref; struct virtio_pmem_req req; struct virtio_pmem_resp resp; @@ -39,6 +42,9 @@ struct virtio_pmem { /* Serialize flush requests to the device. */ struct mutex flush_lock; + /* Complete asynchronous FUA flushes outside the submit path. */ + struct workqueue_struct *flush_wq; + /* nvdimm bus registers virtio pmem device */ struct nvdimm_bus *nvdimm_bus; struct nvdimm_bus_descriptor nd_desc; diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index be3b91b43ea5af..b90812ed74b1ba 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -47,9 +47,6 @@ #define APPLE_ANS_BOOT_STATUS 0x1300 #define APPLE_ANS_BOOT_STATUS_OK 0xde71ce55 -#define APPLE_ANS_UNKNOWN_CTRL 0x24008 -#define APPLE_ANS_PRP_NULL_CHECK BIT(11) - #define APPLE_ANS_LINEAR_SQ_CTRL 0x24908 #define APPLE_ANS_LINEAR_SQ_EN BIT(0) @@ -151,6 +148,23 @@ struct apple_nvme_queue { bool enabled; }; +static inline bool apple_nvme_queue_enabled(struct apple_nvme_queue *q) +{ + /* Pair with apple_nvme_enable_queue(). */ + return smp_load_acquire(&q->enabled); +} + +static inline void apple_nvme_enable_queue(struct apple_nvme_queue *q) +{ + /* Publish queue initialization before setting q->enabled. */ + smp_store_release(&q->enabled, true); +} + +static inline void apple_nvme_disable_queue(struct apple_nvme_queue *q) +{ + WRITE_ONCE(q->enabled, false); +} + /* * The apple_nvme_iod describes the data in an I/O. * @@ -318,13 +332,15 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, u32 tag = nvme_tag_from_cid(cmd->common.command_id); struct apple_nvmmu_tcb *tcb = &q->tcbs[tag]; - tcb->opcode = cmd->common.opcode; + tcb->opcode = 0; tcb->prp1 = cmd->common.dptr.prp1; tcb->prp2 = cmd->common.dptr.prp2; tcb->length = cmd->rw.length; tcb->command_id = tag; - if (nvme_is_write(cmd)) + if (!cmd->common.dptr.prp1) + tcb->dma_flags = 0; + else if (nvme_is_write(cmd)) tcb->dma_flags = APPLE_ANS_TCB_DMA_TO_DEVICE; else tcb->dma_flags = APPLE_ANS_TCB_DMA_FROM_DEVICE; @@ -677,7 +693,7 @@ static bool apple_nvme_handle_cq(struct apple_nvme_queue *q, bool force) bool found; DEFINE_IO_COMP_BATCH(iob); - if (!READ_ONCE(q->enabled) && !force) + if (!apple_nvme_queue_enabled(q) && !force) return false; found = apple_nvme_poll_cq(q, &iob); @@ -780,7 +796,7 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, * We should not need to do this, but we're still using this to * ensure we can drain requests on a dying queue. */ - if (unlikely(!READ_ONCE(q->enabled))) + if (unlikely(!apple_nvme_queue_enabled(q))) return BLK_STS_IOERR; if (!nvme_check_ready(&anv->ctrl, req, true)) @@ -863,7 +879,7 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) nvme_quiesce_io_queues(&anv->ctrl); if (!dead) { - if (READ_ONCE(anv->ioq.enabled)) { + if (apple_nvme_queue_enabled(&anv->ioq)) { apple_nvme_remove_sq(anv); apple_nvme_remove_cq(anv); } @@ -887,8 +903,8 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) nvme_disable_ctrl(&anv->ctrl, false); } - WRITE_ONCE(anv->ioq.enabled, false); - WRITE_ONCE(anv->adminq.enabled, false); + apple_nvme_disable_queue(&anv->ioq); + apple_nvme_disable_queue(&anv->adminq); mb(); /* ensure that nvme_queue_rq() sees that enabled is cleared */ nvme_quiesce_admin_queue(&anv->ctrl); @@ -1016,8 +1032,7 @@ static void apple_nvme_init_queue(struct apple_nvme_queue *q) memset(q->tcbs, 0, anv->hw->max_queue_depth * sizeof(struct apple_nvmmu_tcb)); memset(q->cqes, 0, depth * sizeof(struct nvme_completion)); - WRITE_ONCE(q->enabled, true); - wmb(); /* ensure the first interrupt sees the initialization */ + apple_nvme_enable_queue(q); } static void apple_nvme_reset_work(struct work_struct *work) @@ -1125,17 +1140,6 @@ static void apple_nvme_reset_work(struct work_struct *work) /* Setup the NVMMU for the maximum admin and IO queue depth */ writel(anv->hw->max_queue_depth - 1, anv->mmio_nvme + APPLE_NVMMU_NUM_TCBS); - - /* - * This is probably a chicken bit: without it all commands - * where any PRP is set to zero (including those that don't use - * that field) fail and the co-processor complains about - * "completed with err BAD_CMD-" or a "NULL_PRP_PTR_ERR" in the - * syslog - */ - writel(readl(anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL) & - ~APPLE_ANS_PRP_NULL_CHECK, - anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL); } /* Setup the admin queue */ @@ -1581,7 +1585,8 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) } ret = nvme_init_ctrl(&anv->ctrl, anv->dev, &nvme_ctrl_ops, - NVME_QUIRK_SKIP_CID_GEN | NVME_QUIRK_IDENTIFY_CNS); + NVME_QUIRK_SKIP_CID_GEN | NVME_QUIRK_IDENTIFY_CNS | + NVME_QUIRK_ADMIN_PAGE_ALIGN); if (ret) { dev_err_probe(dev, ret, "Failed to initialize nvme_ctrl"); goto put_dev; @@ -1636,6 +1641,15 @@ static void apple_nvme_remove(struct platform_device *pdev) nvme_stop_ctrl(&anv->ctrl); nvme_remove_namespaces(&anv->ctrl); apple_nvme_disable(anv, true); + if (anv->ctrl.admin_q && !blk_queue_dying(anv->ctrl.admin_q)) { + /* + * If the controller was reset during removal, it's possible + * user requests may be waiting on a stopped queue. Start the + * queue to flush these to completion. + */ + nvme_unquiesce_admin_queue(&anv->ctrl); + blk_mq_destroy_queue(anv->ctrl.admin_q); + } nvme_uninit_ctrl(&anv->ctrl); if (apple_rtkit_is_running(anv->rtk)) { diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 618620e7080e5b..0b80db7a0599f0 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -2071,7 +2071,10 @@ static void nvme_set_ctrl_limits(struct nvme_ctrl *ctrl, lim->max_integrity_segments = ctrl->max_integrity_segments; lim->virt_boundary_mask = ctrl->ops->get_virt_boundary(ctrl, is_admin); lim->max_segment_size = UINT_MAX; - lim->dma_alignment = 3; + if (is_admin && (ctrl->quirks & NVME_QUIRK_ADMIN_PAGE_ALIGN)) + lim->dma_alignment = NVME_CTRL_PAGE_SIZE - 1; + else + lim->dma_alignment = 3; } static bool nvme_update_disk_info(struct nvme_ns *ns, struct nvme_id_ns *id, diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 3082a17320b8f8..48454cb7a0fcc9 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -2100,9 +2100,15 @@ __nvme_fc_init_request(struct nvme_fc_ctrl *ctrl, dev_err(ctrl->dev, "FCP Op failed - rspiu dma mapping failed.\n"); ret = -EFAULT; + goto out_unmap; } atomic_set(&op->state, FCPOP_STATE_IDLE); + return 0; + +out_unmap: + fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.cmddma, + sizeof(op->cmd_iu), DMA_TO_DEVICE); out_on_error: return ret; } diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c index 664216eece4a6b..76596cb2ded2d5 100644 --- a/drivers/nvme/host/ioctl.c +++ b/drivers/nvme/host/ioctl.c @@ -14,45 +14,54 @@ enum { NVME_IOCTL_PARTITION = (1 << 1), }; -static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, - unsigned int flags, bool open_for_write) +static bool nvme_admin_cmd_allowed(struct nvme_ctrl *ctrl, + struct nvme_command *c) { - u32 effects; - - /* - * Do not allow unprivileged passthrough on partitions, as that allows an - * escape from the containment of the partition. - */ - if (flags & NVME_IOCTL_PARTITION) - goto admin; - - /* - * Do not allow unprivileged processes to send vendor specific or fabrics - * commands as we can't be sure about their effects. - */ - if (c->common.opcode >= nvme_cmd_vendor_start || - c->common.opcode == nvme_fabrics_command) - goto admin; - /* * Do not allow unprivileged passthrough of admin commands except * for a subset of identify commands that contain information required * to form proper I/O commands in userspace and do not expose any * potentially sensitive information. */ - if (!ns) { - if (c->common.opcode == nvme_admin_identify) { - switch (c->identify.cns) { - case NVME_ID_CNS_NS: - case NVME_ID_CNS_CS_NS: - case NVME_ID_CNS_NS_CS_INDEP: - case NVME_ID_CNS_CS_CTRL: - case NVME_ID_CNS_CTRL: - return true; - } + switch (c->common.opcode) { + case nvme_admin_identify: + switch (c->identify.cns) { + case NVME_ID_CNS_NS: + case NVME_ID_CNS_CS_NS: + case NVME_ID_CNS_NS_CS_INDEP: + case NVME_ID_CNS_CS_CTRL: + case NVME_ID_CNS_CTRL: + return true; + } + break; + case nvme_admin_set_features: + /* + * Reject Set Features that change controller state the driver + * manages itself; setting them behind the driver's back from + * userspace leaves it unable to react correctly. Keep Alive is + * only armed for fabrics - on other transports it has no + * reserved tag and harms idle power states. + */ + switch (le32_to_cpu(c->features.fid) & 0xff) { + case NVME_FEAT_KATO: + if (ctrl->ops->flags & NVME_F_FABRICS) + break; + fallthrough; + case NVME_FEAT_HOST_BEHAVIOR: + case NVME_FEAT_HOST_MEM_BUF: + case NVME_FEAT_NUM_QUEUES: + case NVME_FEAT_AUTO_PST: + return false; } - goto admin; + break; } + return capable(CAP_SYS_ADMIN); +} + +static bool nvme_ns_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, + bool open_for_write) +{ + u32 effects; /* * Check if the controller provides a Commands Supported and Effects log @@ -61,7 +70,7 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, */ effects = nvme_command_effects(ns->ctrl, ns, c->common.opcode); if (!(effects & NVME_CMD_EFFECTS_CSUPP)) - goto admin; + return capable(CAP_SYS_ADMIN); /* * Don't allow passthrough for command that have intrusive (or unknown) @@ -70,7 +79,7 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_UUID_SEL | NVME_CMD_EFFECTS_SCOPE_MASK)) - goto admin; + return capable(CAP_SYS_ADMIN); /* * Only allow I/O commands that transfer data to the controller or that @@ -79,11 +88,34 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, */ if ((nvme_is_write(c) || (effects & NVME_CMD_EFFECTS_LBCC)) && !open_for_write) - goto admin; + return capable(CAP_SYS_ADMIN); return true; -admin: - return capable(CAP_SYS_ADMIN); +} + +static bool nvme_cmd_allowed(struct nvme_ctrl *ctrl, struct nvme_ns *ns, + struct nvme_command *c, unsigned int flags, + bool open_for_write) +{ + /* + * Do not allow unprivileged passthrough on partitions, as that + * allows an escape from the containment of the partition. + */ + if (flags & NVME_IOCTL_PARTITION) + return capable(CAP_SYS_ADMIN); + + /* + * Do not allow unprivileged processes to send vendor specific or + * fabrics commands as we can't be sure about their effects. + */ + if (c->common.opcode >= nvme_cmd_vendor_start || + c->common.opcode == nvme_fabrics_command) + return capable(CAP_SYS_ADMIN); + + if (!ns) + return nvme_admin_cmd_allowed(ctrl, c); + + return nvme_ns_cmd_allowed(ns, c, open_for_write); } /* @@ -202,7 +234,8 @@ out_free_req: return ret; } -static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) +static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio, + unsigned int flags, bool open_for_write) { struct nvme_user_io io; struct nvme_command c; @@ -260,6 +293,9 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) c.rw.lbat = cpu_to_le16(io.apptag); c.rw.lbatm = cpu_to_le16(io.appmask); + if (!nvme_cmd_allowed(ns->ctrl, ns, &c, flags, open_for_write)) + return -EACCES; + return nvme_submit_user_cmd(ns->queue, &c, io.addr, length, metadata, meta_len, NULL, 0, 0); } @@ -307,7 +343,7 @@ static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns, c.common.cdw14 = cpu_to_le32(cmd.cdw14); c.common.cdw15 = cpu_to_le32(cmd.cdw15); - if (!nvme_cmd_allowed(ns, &c, 0, open_for_write)) + if (!nvme_cmd_allowed(ctrl, ns, &c, 0, open_for_write)) return -EACCES; if (cmd.timeout_ms) @@ -354,7 +390,7 @@ static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns, c.common.cdw14 = cpu_to_le32(cmd.cdw14); c.common.cdw15 = cpu_to_le32(cmd.cdw15); - if (!nvme_cmd_allowed(ns, &c, flags, open_for_write)) + if (!nvme_cmd_allowed(ctrl, ns, &c, flags, open_for_write)) return -EACCES; if (cmd.timeout_ms) @@ -449,6 +485,7 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, const struct nvme_uring_cmd *cmd = io_uring_sqe128_cmd(ioucmd->sqe, struct nvme_uring_cmd); struct request_queue *q = ns ? ns->queue : ctrl->admin_q; + bool open_for_write = ioucmd->file->f_mode & FMODE_WRITE; struct nvme_uring_data d; struct nvme_command c; struct iov_iter iter; @@ -479,7 +516,7 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, c.common.cdw14 = cpu_to_le32(READ_ONCE(cmd->cdw14)); c.common.cdw15 = cpu_to_le32(READ_ONCE(cmd->cdw15)); - if (!nvme_cmd_allowed(ns, &c, 0, ioucmd->file->f_mode & FMODE_WRITE)) + if (!nvme_cmd_allowed(ctrl, ns, &c, 0, open_for_write)) return -EACCES; d.metadata = READ_ONCE(cmd->metadata); @@ -595,7 +632,7 @@ static int nvme_ns_ioctl(struct nvme_ns *ns, unsigned int cmd, case NVME_IOCTL_SUBMIT_IO32: #endif case NVME_IOCTL_SUBMIT_IO: - return nvme_submit_io(ns, argp); + return nvme_submit_io(ns, argp, flags, open_for_write); case NVME_IOCTL_IO64_CMD_VEC: flags |= NVME_IOCTL_VEC; fallthrough; diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 824651cc898dbe..957ded0c6f53b3 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -178,6 +178,11 @@ enum nvme_quirks { * Align dma pool segment size to 512 bytes */ NVME_QUIRK_DMAPOOL_ALIGN_512 = (1 << 22), + + /* + * Admin queue DMA buffers must be page aligned + */ + NVME_QUIRK_ADMIN_PAGE_ALIGN = (1 << 23), }; static inline char *nvme_quirk_name(enum nvme_quirks q) @@ -229,6 +234,8 @@ static inline char *nvme_quirk_name(enum nvme_quirks q) return "broken_msi"; case NVME_QUIRK_DMAPOOL_ALIGN_512: return "dmapool_align_512"; + case NVME_QUIRK_ADMIN_PAGE_ALIGN: + return "admin_page_align"; } return "unknown"; diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index d094717c17a014..664a36eed23c3c 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -213,6 +213,7 @@ static int quirks_param_set(const char *value, const struct kernel_param *kp) if (nvme_parse_quirk_entry(field, &qlist[i])) { pr_err("nvme: failed to parse quirk string %s\n", value); + err = -EINVAL; goto out_free_qlist; } @@ -3839,6 +3840,7 @@ out_disable: nvme_dev_remove_admin(dev); nvme_dbbuf_dma_free(dev); nvme_free_queues(dev, 0); + nvme_release_descriptor_pools(dev); out_release_iod_mempool: mempool_destroy(dev->dmavec_mempool); out_dev_unmap: diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index ab6a0a98dd5da8..7764a3c0195c9f 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -309,8 +309,10 @@ static void nvmet_execute_get_log_page_rmi(struct nvmet_req *req) } log = kzalloc_obj(*log); - if (!log) + if (!log) { + status = NVME_SC_INTERNAL; goto out; + } log->endgid = req->cmd->get_log_page.lsi; disk = req->ns->bdev->bd_disk; @@ -1335,7 +1337,7 @@ static u16 nvmet_set_feat_arbitration(struct nvmet_req *req) void nvmet_execute_set_features(struct nvmet_req *req) { - struct nvmet_subsys *subsys = nvmet_req_subsys(req); + struct nvmet_ctrl *ctrl = nvmet_req_ctrl(req); u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10); u32 cdw11 = le32_to_cpu(req->cmd->common.cdw11); u16 status = 0; @@ -1357,7 +1359,7 @@ void nvmet_execute_set_features(struct nvmet_req *req) break; } nvmet_set_result(req, - (subsys->max_qid - 1) | ((subsys->max_qid - 1) << 16)); + (ctrl->max_qid - 1) | ((ctrl->max_qid - 1) << 16)); break; case NVME_FEAT_IRQ_COALESCE: status = nvmet_set_feat_irq_coalesce(req); @@ -1494,7 +1496,7 @@ void nvmet_get_feat_async_event(struct nvmet_req *req) void nvmet_execute_get_features(struct nvmet_req *req) { - struct nvmet_subsys *subsys = nvmet_req_subsys(req); + struct nvmet_ctrl *ctrl = nvmet_req_ctrl(req); u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10); u16 status = 0; @@ -1534,7 +1536,7 @@ void nvmet_execute_get_features(struct nvmet_req *req) break; case NVME_FEAT_NUM_QUEUES: nvmet_set_result(req, - (subsys->max_qid-1) | ((subsys->max_qid-1) << 16)); + (ctrl->max_qid-1) | ((ctrl->max_qid-1) << 16)); break; case NVME_FEAT_KATO: nvmet_get_feat_kato(req); diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 2b69ffcfc8dfb7..413ee2d16d29ca 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -312,15 +312,17 @@ static ssize_t nvmet_param_mdts_store(struct config_item *item, const char *page, size_t count) { struct nvmet_port *port = to_nvmet_port(item); - int ret; + int ret, mdts; if (nvmet_is_port_enabled(port, __func__)) return -EACCES; - ret = kstrtoint(page, 0, &port->mdts); - if (ret) { - pr_err("Invalid value '%s' for mdts\n", page); + ret = kstrtoint(page, 0, &mdts); + if (ret || mdts < 0 || mdts > NVMET_MAX_MDTS) { + pr_err("Invalid value '%s' for mdts, should be 0-%d\n", + page, NVMET_MAX_MDTS); return -EINVAL; } + port->mdts = mdts; return count; } diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 7e565a84d341af..23fa90ec3c00e0 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -610,7 +610,8 @@ int nvmet_ns_enable(struct nvmet_ns *ns) goto out_dev_put; } - if (percpu_ref_init(&ns->ref, nvmet_destroy_namespace, 0, GFP_KERNEL)) + ret = percpu_ref_init(&ns->ref, nvmet_destroy_namespace, 0, GFP_KERNEL); + if (ret) goto out_pr_exit; nvmet_ns_changed(subsys, ns->nsid); @@ -875,7 +876,7 @@ u16 nvmet_check_cqid(struct nvmet_ctrl *ctrl, u16 cqid, bool create) if (!ctrl->cqs) return NVME_SC_INTERNAL | NVME_STATUS_DNR; - if (cqid > ctrl->subsys->max_qid) + if (cqid > ctrl->max_qid) return NVME_SC_QID_INVALID | NVME_STATUS_DNR; if ((create && ctrl->cqs[cqid]) || (!create && !ctrl->cqs[cqid])) @@ -923,7 +924,7 @@ u16 nvmet_check_sqid(struct nvmet_ctrl *ctrl, u16 sqid, if (!ctrl->sqs) return NVME_SC_INTERNAL | NVME_STATUS_DNR; - if (sqid > ctrl->subsys->max_qid) + if (sqid > ctrl->max_qid) return NVME_SC_QID_INVALID | NVME_STATUS_DNR; if ((create && ctrl->sqs[sqid]) || @@ -1652,11 +1653,29 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) if (!ctrl->changed_ns_list) goto out_free_ctrl; - ctrl->sqs = kzalloc_objs(struct nvmet_sq *, subsys->max_qid + 1); + /* + * Discovery controllers may use some arbitrary high value + * in order to cleanup stale discovery sessions + */ + if (nvmet_is_disc_subsys(ctrl->subsys) && !kato) + kato = NVMET_DISC_KATO_MS; + + /* keep-alive timeout in seconds */ + ctrl->kato = DIV_ROUND_UP(kato, 1000); + + ctrl->err_counter = 0; + spin_lock_init(&ctrl->error_lock); + + down_read(&nvmet_config_sem); + mutex_lock(&subsys->lock); + + ctrl->max_qid = subsys->max_qid; + + ctrl->sqs = kzalloc_objs(struct nvmet_sq *, ctrl->max_qid + 1); if (!ctrl->sqs) goto out_free_changed_ns_list; - ctrl->cqs = kzalloc_objs(struct nvmet_cq *, subsys->max_qid + 1); + ctrl->cqs = kzalloc_objs(struct nvmet_cq *, ctrl->max_qid + 1); if (!ctrl->cqs) goto out_free_sqs; @@ -1669,22 +1688,6 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) } ctrl->cntlid = ret; - /* - * Discovery controllers may use some arbitrary high value - * in order to cleanup stale discovery sessions - */ - if (nvmet_is_disc_subsys(ctrl->subsys) && !kato) - kato = NVMET_DISC_KATO_MS; - - /* keep-alive timeout in seconds */ - ctrl->kato = DIV_ROUND_UP(kato, 1000); - - ctrl->err_counter = 0; - spin_lock_init(&ctrl->error_lock); - - nvmet_start_keep_alive_timer(ctrl); - - mutex_lock(&subsys->lock); ret = nvmet_ctrl_init_pr(ctrl); if (ret) goto init_pr_fail; @@ -1692,6 +1695,9 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) nvmet_setup_p2p_ns_map(ctrl, args->p2p_client); nvmet_debugfs_ctrl_setup(ctrl); mutex_unlock(&subsys->lock); + up_read(&nvmet_config_sem); + + nvmet_start_keep_alive_timer(ctrl); if (args->hostid) uuid_copy(&ctrl->hostid, args->hostid); @@ -1721,14 +1727,14 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) return ctrl; init_pr_fail: - mutex_unlock(&subsys->lock); - nvmet_stop_keep_alive_timer(ctrl); ida_free(&cntlid_ida, ctrl->cntlid); out_free_cqs: kfree(ctrl->cqs); out_free_sqs: kfree(ctrl->sqs); out_free_changed_ns_list: + mutex_unlock(&subsys->lock); + up_read(&nvmet_config_sem); kfree(ctrl->changed_ns_list); out_free_ctrl: kfree(ctrl); diff --git a/drivers/nvme/target/fabrics-cmd-auth.c b/drivers/nvme/target/fabrics-cmd-auth.c index 2b617d3b8bba55..0e88e227edac38 100644 --- a/drivers/nvme/target/fabrics-cmd-auth.c +++ b/drivers/nvme/target/fabrics-cmd-auth.c @@ -30,12 +30,16 @@ void nvmet_auth_sq_init(struct nvmet_sq *sq) sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE; } -static u8 nvmet_auth_negotiate(struct nvmet_req *req, void *d) +static u8 nvmet_auth_negotiate(struct nvmet_req *req, void *d, u32 tl) { struct nvmet_ctrl *ctrl = req->sq->ctrl; struct nvmf_auth_dhchap_negotiate_data *data = d; int i, hash_id = 0, fallback_hash_id = 0, dhgid, fallback_dhgid; + if (tl < sizeof(*data) + + sizeof(struct nvmf_auth_dhchap_protocol_descriptor)) + return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; + pr_debug("%s: ctrl %d qid %d: data sc_d %d napd %d authid %d halen %d dhlen %d\n", __func__, ctrl->cntlid, req->sq->qid, data->sc_c, data->napd, data->auth_protocol[0].dhchap.authid, @@ -71,6 +75,10 @@ static u8 nvmet_auth_negotiate(struct nvmet_req *req, void *d) NVME_AUTH_DHCHAP_AUTH_ID) return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; + if (data->auth_protocol[0].dhchap.dhlen > NVME_AUTH_DHCHAP_MAX_DH_IDS || + data->auth_protocol[0].dhchap.halen > NVME_AUTH_DHCHAP_MAX_HASH_IDS) + return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; + for (i = 0; i < data->auth_protocol[0].dhchap.halen; i++) { u8 host_hmac_id = data->auth_protocol[0].dhchap.idlist[i]; @@ -316,7 +324,7 @@ void nvmet_execute_auth_send(struct nvmet_req *req) } else if (data->auth_id != req->sq->dhchap_step) goto done_failure1; /* Validate negotiation parameters */ - dhchap_status = nvmet_auth_negotiate(req, d); + dhchap_status = nvmet_auth_negotiate(req, d, tl); if (dhchap_status == 0) req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE; diff --git a/drivers/nvme/target/fabrics-cmd.c b/drivers/nvme/target/fabrics-cmd.c index 7cadd1c9e44cc6..42d1d18116715e 100644 --- a/drivers/nvme/target/fabrics-cmd.c +++ b/drivers/nvme/target/fabrics-cmd.c @@ -370,7 +370,7 @@ static void nvmet_execute_io_connect(struct nvmet_req *req) goto out; } - if (unlikely(qid > ctrl->subsys->max_qid)) { + if (unlikely(qid > ctrl->max_qid)) { pr_warn("invalid queue id (%d)\n", qid); status = NVME_SC_CONNECT_INVALID_PARAM | NVME_STATUS_DNR; req->cqe->result.u32 = IPO_IATTR_CONNECT_SQE(qid); diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index 1645f4c818afa2..7750f5b1edbb4d 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -265,6 +265,7 @@ struct nvmet_ctrl { uuid_t hostid; u16 cntlid; + u16 max_qid; u32 kato; struct nvmet_port *port; @@ -753,6 +754,11 @@ static inline struct nvmet_subsys *nvmet_req_subsys(struct nvmet_req *req) return req->sq->ctrl->subsys; } +static inline struct nvmet_ctrl *nvmet_req_ctrl(struct nvmet_req *req) +{ + return req->sq->ctrl; +} + static inline bool nvmet_is_disc_subsys(struct nvmet_subsys *subsys) { return subsys->type != NVME_NQN_NVME; diff --git a/drivers/nvme/target/pci-epf.c b/drivers/nvme/target/pci-epf.c index 346a4badd6b298..803e85df50e54b 100644 --- a/drivers/nvme/target/pci-epf.c +++ b/drivers/nvme/target/pci-epf.c @@ -2081,7 +2081,7 @@ static int nvmet_pci_epf_create_ctrl(struct nvmet_pci_epf *nvme_epf, } /* Allocate our queues, up to the maximum number. */ - ctrl->nr_queues = min(ctrl->tctrl->subsys->max_qid + 1, max_nr_queues); + ctrl->nr_queues = min(ctrl->tctrl->max_qid + 1, max_nr_queues); ret = nvmet_pci_epf_alloc_queues(ctrl); if (ret) goto out_put_ctrl; diff --git a/drivers/nvme/target/pr.c b/drivers/nvme/target/pr.c index c71ae46244ffcf..7d937093b22626 100644 --- a/drivers/nvme/target/pr.c +++ b/drivers/nvme/target/pr.c @@ -355,9 +355,15 @@ static u16 nvmet_pr_replace(struct nvmet_req *req, u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; struct nvmet_ctrl *ctrl = req->sq->ctrl; struct nvmet_pr *pr = &req->ns->pr; - struct nvmet_pr_registrant *reg; + struct nvmet_pr_registrant *reg, *new = NULL; u64 nrkey = le64_to_cpu(d->nrkey); + if (ignore_key && nrkey) { + new = kzalloc_obj(*new); + if (!new) + return NVME_SC_INTERNAL; + } + down(&pr->pr_sem); list_for_each_entry_rcu(reg, &pr->registrant_list, entry) { if (uuid_equal(®->hostid, &ctrl->hostid)) { @@ -365,9 +371,26 @@ static u16 nvmet_pr_replace(struct nvmet_req *req, status = nvmet_pr_update_reg_attr(pr, reg, nvmet_pr_update_reg_rkey, &nrkey); - break; + goto free_data; + } + } + + if (ignore_key) { + if (!nrkey) { + status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; + goto free_data; } + INIT_LIST_HEAD(&new->entry); + new->rkey = nrkey; + uuid_copy(&new->hostid, &ctrl->hostid); + list_add_tail_rcu(&new->entry, &pr->registrant_list); + status = NVME_SC_SUCCESS; + goto out; } + +free_data: + kfree(new); +out: up(&pr->pr_sem); return status; } diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index ea1185b8267ef2..de5a88fbb2337b 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -657,18 +657,25 @@ static void nvmet_rdma_rw_ctx_destroy(struct nvmet_rdma_rsp *rsp) req->sg, req->sg_cnt, nvmet_data_dir(req)); } -static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp) +static void nvmet_rdma_free_rsp_resources(struct nvmet_rdma_rsp *rsp) { struct nvmet_rdma_queue *queue = rsp->queue; - atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail); - if (rsp->n_rdma) nvmet_rdma_rw_ctx_destroy(rsp); if (rsp->req.sg < rsp->cmd->inline_sg || rsp->req.sg >= rsp->cmd->inline_sg + queue->dev->inline_page_count) nvmet_req_free_sgls(&rsp->req); +} + +static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp) +{ + struct nvmet_rdma_queue *queue = rsp->queue; + + atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail); + + nvmet_rdma_free_rsp_resources(rsp); if (unlikely(!list_empty_careful(&queue->rsp_wr_wait_list))) nvmet_rdma_process_wr_wait_list(queue); @@ -1338,9 +1345,27 @@ err_destroy_cq: goto out; } +static bool nvmet_rdma_reclaim_rsp(struct sbitmap *sb, unsigned int bitnr, + void *data) +{ + struct nvmet_rdma_queue *queue = data; + + nvmet_rdma_free_rsp_resources(&queue->rsps[bitnr]); + + return true; +} + static void nvmet_rdma_destroy_queue_ib(struct nvmet_rdma_queue *queue) { ib_drain_qp(queue->qp); + + /* + * Reclaim resources of a response that is still in-flight when the + * queue is being torn down. This happens when the connection was + * forcefully disconnected while an I/O is in flight. + */ + sbitmap_for_each_set(&queue->rsp_tags, nvmet_rdma_reclaim_rsp, queue); + if (queue->cm_id) rdma_destroy_id(queue->cm_id); ib_destroy_qp(queue->qp); diff --git a/drivers/nvme/target/zns.c b/drivers/nvme/target/zns.c index f00921931eb64d..a13befd5f3adf9 100644 --- a/drivers/nvme/target/zns.c +++ b/drivers/nvme/target/zns.c @@ -116,7 +116,7 @@ void nvmet_execute_identify_ns_zns(struct nvmet_req *req) mutex_unlock(&req->ns->subsys->lock); } - if (!bdev_is_zoned(req->ns->bdev)) { + if (!req->ns->bdev || !bdev_is_zoned(req->ns->bdev)) { status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; req->error_loc = offsetof(struct nvme_identify, nsid); goto out; diff --git a/drivers/of/irq.c b/drivers/of/irq.c index 967c191008797f..ec035367c9500c 100644 --- a/drivers/of/irq.c +++ b/drivers/of/irq.c @@ -563,7 +563,9 @@ const struct cpumask *of_irq_get_affinity(struct device_node *dev, int index) of_phandle_args_to_fwspec(oirq.np, oirq.args, oirq.args_count, &fwspec); - if (irq_populate_fwspec_info(&fwspec, &info)) + rc = irq_populate_fwspec_info(&fwspec, &info); + of_node_put(oirq.np); + if (rc) return NULL; return info.affinity; diff --git a/drivers/opp/core.c b/drivers/opp/core.c index ab0b0a2f85a178..b6966e509f7dd4 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -1412,13 +1412,12 @@ static int _set_opp(struct device *dev, struct opp_table *opp_table, */ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) { + struct opp_table *opp_table __free(put_opp_table) = + _find_opp_table(dev); struct dev_pm_opp *opp __free(put_opp) = NULL; unsigned long freq = 0, temp_freq; bool forced = false; - struct opp_table *opp_table __free(put_opp_table) = - _find_opp_table(dev); - if (IS_ERR(opp_table)) { dev_err(dev, "%s: device's opp table doesn't exist\n", __func__); return PTR_ERR(opp_table); @@ -2870,11 +2869,10 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_add_dynamic); static int _opp_set_availability(struct device *dev, unsigned long freq, bool availability_req) { - struct dev_pm_opp *opp __free(put_opp) = ERR_PTR(-ENODEV), *tmp_opp; - /* Find the opp_table */ struct opp_table *opp_table __free(put_opp_table) = _find_opp_table(dev); + struct dev_pm_opp *opp __free(put_opp) = ERR_PTR(-ENODEV), *tmp_opp; if (IS_ERR(opp_table)) { dev_warn(dev, "%s: Device OPP not found (%ld)\n", __func__, @@ -2932,12 +2930,11 @@ int dev_pm_opp_adjust_voltage(struct device *dev, unsigned long freq, unsigned long u_volt_max) { - struct dev_pm_opp *opp __free(put_opp) = ERR_PTR(-ENODEV), *tmp_opp; - int r; - /* Find the opp_table */ struct opp_table *opp_table __free(put_opp_table) = _find_opp_table(dev); + struct dev_pm_opp *opp __free(put_opp) = ERR_PTR(-ENODEV), *tmp_opp; + int r; if (IS_ERR(opp_table)) { r = PTR_ERR(opp_table); diff --git a/drivers/pci/controller/cadence/Kconfig b/drivers/pci/controller/cadence/Kconfig index 9e651d54597353..fd704dec47dda5 100644 --- a/drivers/pci/controller/cadence/Kconfig +++ b/drivers/pci/controller/cadence/Kconfig @@ -20,11 +20,12 @@ config PCIE_CADENCE_EP config PCIE_CADENCE_PLAT tristate + select PCIE_CADENCE_HOST if PCIE_CADENCE_PLAT_HOST != n + select PCIE_CADENCE_EP if PCIE_CADENCE_PLAT_EP != n config PCIE_CADENCE_PLAT_HOST tristate "Cadence platform PCIe controller (host mode)" depends on OF - select PCIE_CADENCE_HOST select PCIE_CADENCE_PLAT help Say Y here if you want to support the Cadence PCIe platform controller in @@ -35,7 +36,6 @@ config PCIE_CADENCE_PLAT_EP tristate "Cadence platform PCIe controller (endpoint mode)" depends on OF depends on PCI_ENDPOINT - select PCIE_CADENCE_EP select PCIE_CADENCE_PLAT help Say Y here if you want to support the Cadence PCIe platform controller in diff --git a/drivers/pci/controller/cadence/pci-j721e.c b/drivers/pci/controller/cadence/pci-j721e.c index ae916e7b1927a4..822602836b1207 100644 --- a/drivers/pci/controller/cadence/pci-j721e.c +++ b/drivers/pci/controller/cadence/pci-j721e.c @@ -383,7 +383,7 @@ static const struct j721e_pcie_data j7200_pcie_rc_data = { .quirk_detect_quiet_flag = true, .linkdown_irq_regfield = J7200_LINK_DOWN, .byte_access_allowed = true, - .max_lanes = 2, + .max_lanes = 4, }; static const struct j721e_pcie_data j7200_pcie_ep_data = { @@ -391,7 +391,7 @@ static const struct j721e_pcie_data j7200_pcie_ep_data = { .quirk_detect_quiet_flag = true, .linkdown_irq_regfield = J7200_LINK_DOWN, .quirk_disable_flr = true, - .max_lanes = 2, + .max_lanes = 4, }; static const struct j721e_pcie_data am64_pcie_rc_data = { diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig index aa0b784c85b474..49a7a2c50ca153 100644 --- a/drivers/pci/controller/dwc/Kconfig +++ b/drivers/pci/controller/dwc/Kconfig @@ -126,7 +126,9 @@ config PCI_IMX6_EP depends on ARCH_MXC || COMPILE_TEST depends on PCI_ENDPOINT select PCIE_DW_EP + select PCI_HOST_COMMON select PCI_IMX6 + select PCI_PWRCTRL_GENERIC help Enables support for the PCIe controller in the i.MX SoCs to work in endpoint mode. The PCI controller on i.MX is based diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c index 278d2dba1db08d..f1b27aed488dea 100644 --- a/drivers/pci/controller/dwc/pci-keystone.c +++ b/drivers/pci/controller/dwc/pci-keystone.c @@ -1389,13 +1389,17 @@ static int ks_pcie_fault(unsigned long addr, unsigned int fsr, static int __init ks_pcie_init(void) { + struct device_node *np; /* * PCIe access errors that result into OCP errors are caught by ARM as * "External aborts" */ - if (of_find_matching_node(NULL, ks_pcie_of_match)) + np = of_find_matching_node(NULL, ks_pcie_of_match); + if (np) { + of_node_put(np); hook_fault_code(17, ks_pcie_fault, SIGBUS, 0, "Asynchronous external abort"); + } return platform_driver_register(&ks_pcie_driver); } diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index 7d2794945704e1..0b915824963af6 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -1032,6 +1032,8 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, * there is no unified way to check if we have operations in * flight, thus we don't know if we should WARN() or not. */ + /* flush posted write before unmap */ + readl(ep->msi_mem + ep->msi_iatu_mapped_offset); dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); ep->msi_iatu_mapped = false; } @@ -1044,6 +1046,7 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, return ret; ep->msi_iatu_mapped = true; + ep->msi_iatu_mapped_offset = offset; ep->msi_msg_addr = msg_addr; ep->msi_map_size = map_size; } @@ -1124,6 +1127,17 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, return -EPERM; } + /* + * ep->msi_iatu_mapped means that an MSI target address is cached, + * unmap it first so that we can reuse ep->msi_mem_phys for MSI-X. + */ + if (ep->msi_iatu_mapped) { + /* flush posted write before unmap */ + readl(ep->msi_mem + ep->msi_iatu_mapped_offset); + dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); + ep->msi_iatu_mapped = false; + } + msg_addr = dw_pcie_ep_align_addr(epc, msg_addr, &map_size, &offset); ret = dw_pcie_ep_map_addr(epc, func_no, 0, ep->msi_mem_phys, msg_addr, map_size); @@ -1153,6 +1167,11 @@ void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep) { struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + if (ep->msi_iatu_mapped) { + dw_pcie_ep_unmap_addr(ep->epc, 0, 0, ep->msi_mem_phys); + ep->msi_iatu_mapped = false; + } + dwc_pcie_debugfs_deinit(pci); dw_pcie_edma_remove(pci); } diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index de4b245b1758c3..1d21394741572b 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -518,6 +518,7 @@ struct dw_pcie_ep { /* MSI outbound iATU state */ bool msi_iatu_mapped; + size_t msi_iatu_mapped_offset; u64 msi_msg_addr; size_t msi_map_size; }; diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index d8eb52857f69c0..d62cf302de2b88 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -1820,6 +1820,23 @@ static const struct pci_ecam_ops pci_qcom_ecam_ops = { } }; +/* Check if @node is a child of @dev in DT */ +static bool qcom_pcie_is_child_node(struct device *dev, + struct device_node *node) +{ + struct device_node *parent; + + for (parent = of_get_parent(node); parent; + parent = of_get_next_parent(parent)) { + if (parent == dev->of_node) { + of_node_put(parent); + return true; + } + } + + return false; +} + /* Parse PERST# from all nodes in depth first manner starting from @np */ static int qcom_pcie_parse_perst(struct qcom_pcie *pcie, struct qcom_pcie_port *port, @@ -1827,6 +1844,7 @@ static int qcom_pcie_parse_perst(struct qcom_pcie *pcie, { struct device *dev = pcie->pci->dev; struct qcom_pcie_perst *perst; + struct device_node *gpio_np; struct gpio_desc *reset; int ret; @@ -1840,6 +1858,25 @@ static int qcom_pcie_parse_perst(struct qcom_pcie *pcie, if (!of_find_property(np, "reset-gpios", NULL)) goto parse_child_node; + /* + * Skip GPIOs provided by a PCIe device which is a child of the Root + * Complex (e.g., a PCIe switch with GPIO controller capability). Such + * controllers won't be available at RC probe time and their PERST# + * should be controlled by the respective PCI client driver + * implementation. + */ + gpio_np = of_parse_phandle(np, "reset-gpios", 0); + if (!gpio_np) { + dev_err(dev, "Failed to parse GPIO provider\n"); + return -EINVAL; + } + + if (qcom_pcie_is_child_node(dev, gpio_np)) { + of_node_put(gpio_np); + goto parse_child_node; + } + of_node_put(gpio_np); + reset = devm_fwnode_gpiod_get(dev, of_fwnode_handle(np), "reset", GPIOD_OUT_HIGH, "PERST#"); if (IS_ERR(reset)) { diff --git a/drivers/pci/controller/dwc/pcie-ultrarisc.c b/drivers/pci/controller/dwc/pcie-ultrarisc.c index 6ee661ceff67a3..8e57135453b7ca 100644 --- a/drivers/pci/controller/dwc/pcie-ultrarisc.c +++ b/drivers/pci/controller/dwc/pcie-ultrarisc.c @@ -5,6 +5,7 @@ * Copyright (C) 2026 UltraRISC Technology (Shanghai) Co., Ltd. */ +#include #include #include #include @@ -23,6 +24,12 @@ #define ULTRARISC_PCIE_COMP_TIMEOUT_65_210MS 0x6 +struct ultrarisc_pcie { + struct dw_pcie pci; + struct clk_bulk_data *clks; + int num_clks; +}; + static struct pci_ops ultrarisc_pci_ops = { .map_bus = dw_pcie_own_conf_map_bus, .read = pci_generic_config_read32, @@ -98,17 +105,47 @@ static const struct dw_pcie_ops dw_pcie_ops = { .start_link = ultrarisc_pcie_start_link, }; +static int ultrarisc_pcie_enable_clks(struct ultrarisc_pcie *ultra) +{ + return clk_bulk_prepare_enable(ultra->num_clks, ultra->clks); +} + +static void ultrarisc_pcie_disable_clks(void *data) +{ + struct ultrarisc_pcie *ultra = data; + + clk_bulk_disable_unprepare(ultra->num_clks, ultra->clks); +} + +static int ultrarisc_pcie_init_clks(struct ultrarisc_pcie *ultra) +{ + struct device *dev = ultra->pci.dev; + int ret; + + ultra->num_clks = devm_clk_bulk_get_all(dev, &ultra->clks); + if (ultra->num_clks < 0) + return dev_err_probe(dev, ultra->num_clks, "Failed to get clocks\n"); + + ret = ultrarisc_pcie_enable_clks(ultra); + if (ret) + return dev_err_probe(dev, ret, "Failed to enable clocks\n"); + + return devm_add_action_or_reset(dev, ultrarisc_pcie_disable_clks, ultra); +} + static int ultrarisc_pcie_probe(struct platform_device *pdev) { + struct ultrarisc_pcie *ultra; struct device *dev = &pdev->dev; struct dw_pcie_rp *pp; struct dw_pcie *pci; int ret; - pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); - if (!pci) + ultra = devm_kzalloc(dev, sizeof(*ultra), GFP_KERNEL); + if (!ultra) return -ENOMEM; + pci = &ultra->pci; pci->dev = dev; pci->ops = &dw_pcie_ops; @@ -117,7 +154,11 @@ static int ultrarisc_pcie_probe(struct platform_device *pdev) pp = &pci->pp; - platform_set_drvdata(pdev, pci); + platform_set_drvdata(pdev, ultra); + + ret = ultrarisc_pcie_init_clks(ultra); + if (ret) + return ret; pp->num_vectors = MAX_MSI_IRQS; /* No L2/L3 Ready indication is available on this platform */ @@ -135,16 +176,46 @@ static int ultrarisc_pcie_probe(struct platform_device *pdev) static int ultrarisc_pcie_suspend_noirq(struct device *dev) { - struct dw_pcie *pci = dev_get_drvdata(dev); + struct ultrarisc_pcie *ultra = dev_get_drvdata(dev); + struct dw_pcie *pci = &ultra->pci; + int ret; + + /* + * A failed resume leaves the DWC suspended and the clocks disabled. + * A later suspend must not access the controller or disable them again. + */ + if (pci->suspended) + return 0; + + ret = dw_pcie_suspend_noirq(pci); + if (ret) + return ret; + + if (pci->suspended) + ultrarisc_pcie_disable_clks(ultra); - return dw_pcie_suspend_noirq(pci); + return 0; } static int ultrarisc_pcie_resume_noirq(struct device *dev) { - struct dw_pcie *pci = dev_get_drvdata(dev); + struct ultrarisc_pcie *ultra = dev_get_drvdata(dev); + struct dw_pcie *pci = &ultra->pci; + int ret; - return dw_pcie_resume_noirq(pci); + if (pci->suspended) { + ret = ultrarisc_pcie_enable_clks(ultra); + if (ret) + return ret; + + ret = dw_pcie_resume_noirq(pci); + if (ret) { + ultrarisc_pcie_disable_clks(ultra); + return ret; + } + } + + return 0; } static const struct dev_pm_ops ultrarisc_pcie_pm_ops = { diff --git a/drivers/pci/controller/pci-xgene.c b/drivers/pci/controller/pci-xgene.c index b95afa35201d0f..83c9a2930eecaf 100644 --- a/drivers/pci/controller/pci-xgene.c +++ b/drivers/pci/controller/pci-xgene.c @@ -58,7 +58,6 @@ #define XGENE_PCIE_IP_VER_2 2 struct xgene_pcie { - struct device_node *node; struct device *dev; struct clk *clk; void __iomem *csr_base; @@ -526,7 +525,7 @@ static void xgene_pcie_setup_ib_reg(struct xgene_pcie *port, static int xgene_pcie_parse_map_dma_ranges(struct xgene_pcie *port) { - struct device_node *np = port->node; + struct device_node *np = port->dev->of_node; struct of_pci_range range; struct of_pci_range_parser parser; struct device *dev = port->dev; @@ -612,7 +611,6 @@ static bool xgene_check_pcie_msi_ready(void) static int xgene_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct device_node *dn = dev->of_node; struct xgene_pcie *port; struct pci_host_bridge *bridge; int ret; @@ -627,7 +625,6 @@ static int xgene_pcie_probe(struct platform_device *pdev) port = pci_host_bridge_priv(bridge); - port->node = of_node_get(dn); port->dev = dev; port->version = XGENE_PCIE_IP_VER_1; diff --git a/drivers/pci/controller/plda/pcie-starfive.c b/drivers/pci/controller/plda/pcie-starfive.c index 628f8c8d67471d..fab44054a5de2f 100644 --- a/drivers/pci/controller/plda/pcie-starfive.c +++ b/drivers/pci/controller/plda/pcie-starfive.c @@ -419,7 +419,11 @@ static int starfive_pcie_probe(struct platform_device *pdev) return ret; pm_runtime_enable(&pdev->dev); - pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); + if (ret < 0) { + pm_runtime_disable(&pdev->dev); + return dev_err_probe(dev, ret, "failed to resume device\n"); + } plda->host_ops = &sf_host_ops; plda->num_events = PLDA_MAX_EVENT_NUM; @@ -445,9 +449,9 @@ static void starfive_pcie_remove(struct platform_device *pdev) { struct starfive_jh7110_pcie *pcie = platform_get_drvdata(pdev); - pm_runtime_put(&pdev->dev); - pm_runtime_disable(&pdev->dev); plda_pcie_host_deinit(&pcie->plda); + pm_runtime_put_sync(&pdev->dev); + pm_runtime_disable(&pdev->dev); platform_set_drvdata(pdev, NULL); } diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index f36778e62ac1a5..e16aa59dd7ac8b 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -29,6 +29,47 @@ struct pci_dynid { struct pci_device_id id; }; +/** + * do_pci_add_dynid - Add a new PCI device ID to this driver and re-probe + * @drv: target PCI driver + * @id: ID to be added + * @check_dup: whether to check if matching ID is already present + * + * Add a new dynamic PCI device ID to this driver and causes the driver to + * probe for all devices again. @drv must have been registered prior to calling + * this function. + * + * Context: Does GFP_KERNEL allocation. + * + * Return: 0 on success, -errno on failure. + */ +static int do_pci_add_dynid(struct pci_driver *drv, + const struct pci_device_id *id, + bool check_dup) +{ + struct pci_dynid *dynid, *existing_dynid; + + dynid = kzalloc_obj(*dynid); + if (!dynid) + return -ENOMEM; + + dynid->id = *id; + + scoped_guard(spinlock, &drv->dynids.lock) { + if (check_dup) { + list_for_each_entry(existing_dynid, &drv->dynids.list, node) { + if (pci_match_one_id(&existing_dynid->id, id)) { + kfree(dynid); + return -EEXIST; + } + } + } + list_add_tail(&dynid->node, &drv->dynids.list); + } + + return driver_attach(&drv->driver); +} + /** * pci_add_dynid - add a new PCI device ID to this driver and re-probe devices * @drv: target pci driver @@ -56,25 +97,17 @@ int pci_add_dynid(struct pci_driver *drv, unsigned int class, unsigned int class_mask, unsigned long driver_data) { - struct pci_dynid *dynid; - - dynid = kzalloc_obj(*dynid); - if (!dynid) - return -ENOMEM; - - dynid->id.vendor = vendor; - dynid->id.device = device; - dynid->id.subvendor = subvendor; - dynid->id.subdevice = subdevice; - dynid->id.class = class; - dynid->id.class_mask = class_mask; - dynid->id.driver_data = driver_data; - - spin_lock(&drv->dynids.lock); - list_add_tail(&dynid->node, &drv->dynids.list); - spin_unlock(&drv->dynids.lock); - - return driver_attach(&drv->driver); + struct pci_device_id id = { + .vendor = vendor, + .device = device, + .subvendor = subvendor, + .subdevice = subdevice, + .class = class, + .class_mask = class_mask, + .driver_data = driver_data, + }; + + return do_pci_add_dynid(drv, &id, false); } EXPORT_SYMBOL_GPL(pci_add_dynid); @@ -90,6 +123,31 @@ static void pci_free_dynids(struct pci_driver *drv) spin_unlock(&drv->dynids.lock); } +/** + * do_pci_match_id - See if a PCI ID matches a given pci_id table + * @ids: array of PCI device ID structures to search in + * @dev_id: the actual PCI device ID structure to match against. + * @include_override_only: also match against device ID entries marked as + * override only. + * + * Return: the matching pci_device_id structure or %NULL if there is no match. + */ +static const struct pci_device_id * +do_pci_match_id(const struct pci_device_id *ids, + const struct pci_device_id *dev_id, + bool include_override_only) +{ + if (ids) { + while (ids->vendor || ids->subvendor || ids->class_mask) { + if ((!ids->override_only || include_override_only) && + pci_match_one_id(ids, dev_id)) + return ids; + ids++; + } + } + return NULL; +} + /** * pci_match_id - See if a PCI device matches a given pci_id table * @ids: array of PCI device ID structures to search in @@ -105,14 +163,9 @@ static void pci_free_dynids(struct pci_driver *drv) const struct pci_device_id *pci_match_id(const struct pci_device_id *ids, struct pci_dev *dev) { - if (ids) { - while (ids->vendor || ids->subvendor || ids->class_mask) { - if (pci_match_one_device(ids, dev)) - return ids; - ids++; - } - } - return NULL; + struct pci_device_id dev_id = pci_id_from_device(dev); + + return do_pci_match_id(ids, &dev_id, true); } EXPORT_SYMBOL(pci_match_id); @@ -127,6 +180,7 @@ static const struct pci_device_id pci_device_id_any = { * pci_match_device - See if a device matches a driver's list of IDs * @drv: the PCI driver to match against * @dev: the PCI device structure to match against + * @id_copy: place to store copy of pci_device_id for dynamic ID * * Used by a driver to check whether a PCI device is in its list of * supported devices or in the dynids list, which may have been augmented @@ -134,10 +188,11 @@ static const struct pci_device_id pci_device_id_any = { * structure or %NULL if there is no match. */ static const struct pci_device_id *pci_match_device(struct pci_driver *drv, - struct pci_dev *dev) + struct pci_dev *dev, + struct pci_device_id *id_copy) { - struct pci_dynid *dynid; - const struct pci_device_id *found_id = NULL, *ids; + const struct pci_device_id *found_id = NULL; + struct pci_device_id dev_id; int ret; /* When driver_override is set, only bind to the matching driver */ @@ -145,45 +200,29 @@ static const struct pci_device_id *pci_match_device(struct pci_driver *drv, if (ret == 0) return NULL; + dev_id = pci_id_from_device(dev); /* Look at the dynamic ids first, before the static ones */ - spin_lock(&drv->dynids.lock); - list_for_each_entry(dynid, &drv->dynids.list, node) { - if (pci_match_one_device(&dynid->id, dev)) { - found_id = &dynid->id; - break; + scoped_guard(spinlock, &drv->dynids.lock) { + struct pci_dynid *dynid; + + list_for_each_entry(dynid, &drv->dynids.list, node) { + if (pci_match_one_id(&dynid->id, &dev_id)) { + *id_copy = dynid->id; + return id_copy; + } } } - spin_unlock(&drv->dynids.lock); + found_id = do_pci_match_id(drv->id_table, &dev_id, ret > 0); if (found_id) return found_id; - for (ids = drv->id_table; (found_id = pci_match_id(ids, dev)); - ids = found_id + 1) { - /* - * The match table is split based on driver_override. - * In case override_only was set, enforce driver_override - * matching. - */ - if (found_id->override_only) { - if (ret > 0) - return found_id; - } else { - return found_id; - } - } - /* driver_override will always match, send a dummy id */ if (ret > 0) return &pci_device_id_any; return NULL; } -static void _pci_free_device(struct device *dev) -{ - kfree(to_pci_dev(dev)); -} - /** * new_id_store - sysfs frontend to pci_add_dynid() * @driver: target device driver @@ -197,38 +236,22 @@ static ssize_t new_id_store(struct device_driver *driver, const char *buf, { struct pci_driver *pdrv = to_pci_driver(driver); const struct pci_device_id *ids = pdrv->id_table; - u32 vendor, device, subvendor = PCI_ANY_ID, - subdevice = PCI_ANY_ID, class = 0, class_mask = 0; - unsigned long driver_data = 0; + struct pci_device_id id = { + .subvendor = PCI_ANY_ID, + .subdevice = PCI_ANY_ID + }; int fields; int retval = 0; fields = sscanf(buf, "%x %x %x %x %x %x %lx", - &vendor, &device, &subvendor, &subdevice, - &class, &class_mask, &driver_data); + &id.vendor, &id.device, &id.subvendor, &id.subdevice, + &id.class, &id.class_mask, &id.driver_data); if (fields < 2) return -EINVAL; if (fields != 7) { - struct pci_dev *pdev = kzalloc_obj(*pdev); - if (!pdev) - return -ENOMEM; - - pdev->vendor = vendor; - pdev->device = device; - pdev->subsystem_vendor = subvendor; - pdev->subsystem_device = subdevice; - pdev->class = class; - pdev->dev.release = _pci_free_device; - - device_initialize(&pdev->dev); - if (pci_match_device(pdrv, pdev)) - retval = -EEXIST; - - put_device(&pdev->dev); - - if (retval) - return retval; + if (do_pci_match_id(pdrv->id_table, &id, false)) + return -EEXIST; } /* Only accept driver_data values that match an existing id_table @@ -236,7 +259,7 @@ static ssize_t new_id_store(struct device_driver *driver, const char *buf, if (ids) { retval = -EINVAL; while (ids->vendor || ids->subvendor || ids->class_mask) { - if (driver_data == ids->driver_data) { + if (id.driver_data == ids->driver_data) { retval = 0; break; } @@ -246,8 +269,7 @@ static ssize_t new_id_store(struct device_driver *driver, const char *buf, return retval; } - retval = pci_add_dynid(pdrv, vendor, device, subvendor, subdevice, - class, class_mask, driver_data); + retval = do_pci_add_dynid(pdrv, &id, fields != 7); if (retval) return retval; return count; @@ -445,12 +467,13 @@ void pci_probe_flush_workqueue(void) static int __pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev) { const struct pci_device_id *id; + struct pci_device_id id_copy; int error = 0; if (drv->probe) { error = -ENODEV; - id = pci_match_device(drv, pci_dev); + id = pci_match_device(drv, pci_dev, &id_copy); if (id) error = pci_call_probe(drv, pci_dev, id); } @@ -1538,12 +1561,13 @@ static int pci_bus_match(struct device *dev, const struct device_driver *drv) struct pci_dev *pci_dev = to_pci_dev(dev); struct pci_driver *pci_drv; const struct pci_device_id *found_id; + struct pci_device_id id_copy; if (pci_dev_binding_disallowed(pci_dev)) return 0; pci_drv = (struct pci_driver *)to_pci_driver(drv); - found_id = pci_match_device(pci_drv, pci_dev); + found_id = pci_match_device(pci_drv, pci_dev, &id_copy); if (found_id) return 1; diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index 1be627913a9bec..6b016792e0ad82 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -967,6 +967,11 @@ static int pci_mmap_legacy_mem(struct file *filp, struct kobject *kobj, struct vm_area_struct *vma) { struct pci_bus *bus = to_pci_bus(kobj_to_dev(kobj)); + int ret; + + ret = security_locked_down(LOCKDOWN_PCI_ACCESS); + if (ret) + return ret; return pci_mmap_legacy_page_range(bus, vma, pci_mmap_mem); } @@ -987,6 +992,11 @@ static int pci_mmap_legacy_io(struct file *filp, struct kobject *kobj, struct vm_area_struct *vma) { struct pci_bus *bus = to_pci_bus(kobj_to_dev(kobj)); + int ret; + + ret = security_locked_down(LOCKDOWN_PCI_ACCESS); + if (ret) + return ret; return pci_mmap_legacy_page_range(bus, vma, pci_mmap_io); } @@ -1003,6 +1013,11 @@ static inline umode_t __pci_legacy_is_visible(struct kobject *kobj, bool sparse) { struct pci_bus *bus = to_pci_bus(kobj_to_dev(kobj)); + int ret; + + ret = security_locked_down(LOCKDOWN_PCI_ACCESS); + if (ret) + return ret; if (pci_legacy_has_sparse(bus, type) != sparse) return 0; @@ -1274,7 +1289,16 @@ static loff_t pci_llseek_resource(struct file *filep, .llseek = pci_llseek_resource, \ .mmap = pci_mmap_resource_uc, #else -# define __PCI_RESOURCE_IO_MMAP_ATTRS +static int pci_mmap_resource_io_unsupported(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + struct vm_area_struct *vma) +{ + return -EINVAL; +} + +# define __PCI_RESOURCE_IO_MMAP_ATTRS \ + .mmap = pci_mmap_resource_io_unsupported, #endif #define pci_dev_resource_io_attr(_bar) \ diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 4469e1a77f3c13..62c1b324a9bd03 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -442,21 +442,40 @@ static inline int pci_setup_cardbus(char *str) { return -ENOENT; } #endif /* CONFIG_CARDBUS */ /** - * pci_match_one_device - Tell if a PCI device structure has a matching - * PCI device id structure - * @id: single PCI device id structure to match - * @dev: the PCI device structure to match against + * pci_id_from_device - Obtain a pci_device_id from a PCI device + * @dev: the PCI device * - * Returns the matching pci_device_id structure or %NULL if there is no match. + * Return: a pci_device_id filled. */ -static inline const struct pci_device_id * -pci_match_one_device(const struct pci_device_id *id, const struct pci_dev *dev) +static inline struct pci_device_id pci_id_from_device(const struct pci_dev *dev) { - if ((id->vendor == PCI_ANY_ID || id->vendor == dev->vendor) && - (id->device == PCI_ANY_ID || id->device == dev->device) && - (id->subvendor == PCI_ANY_ID || id->subvendor == dev->subsystem_vendor) && - (id->subdevice == PCI_ANY_ID || id->subdevice == dev->subsystem_device) && - !((id->class ^ dev->class) & id->class_mask)) + return (struct pci_device_id) { + .vendor = dev->vendor, + .device = dev->device, + .subvendor = dev->subsystem_vendor, + .subdevice = dev->subsystem_device, + .class = dev->class, + }; +} + +/** + * pci_match_one_id - Tell if a PCI device ID matches a needle PCI device ID + * @id: single PCI device id structure to match against (needle) + * @dev_id: the actual ID from the PCI device + * + * ID can be retrieved from device using pci_id_from_device(). + * + * Return: the matching pci_device_id structure or %NULL if there is no match. + */ +static inline const struct pci_device_id * +pci_match_one_id(const struct pci_device_id *id, + const struct pci_device_id *dev_id) +{ + if ((id->vendor == PCI_ANY_ID || id->vendor == dev_id->vendor) && + (id->device == PCI_ANY_ID || id->device == dev_id->device) && + (id->subvendor == PCI_ANY_ID || id->subvendor == dev_id->subvendor) && + (id->subdevice == PCI_ANY_ID || id->subdevice == dev_id->subdevice) && + !((id->class ^ dev_id->class) & id->class_mask)) return id; return NULL; } diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index 172783e7f519ab..9b3548544a85ec 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -861,6 +861,7 @@ static void pcie_aspm_override_default_link_state(struct pcie_link_state *link) static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) { struct pci_dev *child = link->downstream, *parent = link->pdev; + struct pci_dev *fn; u16 parent_lnkctl, child_lnkctl; struct pci_bus *linkbus = parent->subordinate; @@ -894,10 +895,11 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) /* Disable L0s/L1 before updating L1SS config */ if (FIELD_GET(PCI_EXP_LNKCTL_ASPMC, child_lnkctl) || FIELD_GET(PCI_EXP_LNKCTL_ASPMC, parent_lnkctl)) { - pcie_capability_write_word(child, PCI_EXP_LNKCTL, - child_lnkctl & ~PCI_EXP_LNKCTL_ASPMC); - pcie_capability_write_word(parent, PCI_EXP_LNKCTL, - parent_lnkctl & ~PCI_EXP_LNKCTL_ASPMC); + list_for_each_entry(fn, &linkbus->devices, bus_list) + pcie_capability_clear_and_set_word(fn, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, 0); + pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, 0); } /* @@ -927,8 +929,13 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) /* Restore L0s/L1 if they were enabled */ if (FIELD_GET(PCI_EXP_LNKCTL_ASPMC, child_lnkctl) || FIELD_GET(PCI_EXP_LNKCTL_ASPMC, parent_lnkctl)) { - pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_lnkctl); - pcie_capability_write_word(child, PCI_EXP_LNKCTL, child_lnkctl); + pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, + parent_lnkctl & PCI_EXP_LNKCTL_ASPMC); + list_for_each_entry(fn, &linkbus->devices, bus_list) + pcie_capability_clear_and_set_word(fn, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, + child_lnkctl & PCI_EXP_LNKCTL_ASPMC); } /* Save default state */ diff --git a/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c b/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c index 1555e8a9b3ca1d..b0efe7560c9144 100644 --- a/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c +++ b/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c @@ -240,12 +240,18 @@ static int tc9563_pwrctrl_disable_port(struct tc9563_pwrctrl *tc9563, if (!cfg->disable_port) return 0; - if (port == TC9563_DSP1) { + switch (port) { + case TC9563_DSP1: seq = dsp1_pwroff_seq; len = ARRAY_SIZE(dsp1_pwroff_seq); - } else { + break; + case TC9563_DSP2: seq = dsp2_pwroff_seq; len = ARRAY_SIZE(dsp2_pwroff_seq); + break; + default: + /* Only external downstream ports DSP1/DSP2 can be powered off */ + return 0; } ret = tc9563_pwrctrl_i2c_bulk_write(tc9563->client, seq, len); @@ -595,12 +601,18 @@ static int tc9563_pwrctrl_probe(struct platform_device *pdev) ret = tc9563_pwrctrl_parse_device_dt(tc9563, child, port); if (ret) break; - /* Embedded ethernet device are under DSP3 */ + + /* + * The integrated Ethernet MAC Endpoint under DSP3 is a single + * device whose functions share the same config registers. + */ if (port == TC9563_DSP3) { - for_each_child_of_node_scoped(child, child1) { - port++; + struct device_node *eth __free(device_node) = + of_get_next_available_child(child, NULL); + + if (eth) { ret = tc9563_pwrctrl_parse_device_dt(tc9563, - child1, port); + eth, TC9563_ETHERNET); if (ret) break; } diff --git a/drivers/pci/search.c b/drivers/pci/search.c index e3d3177fce549e..34f8de551d5878 100644 --- a/drivers/pci/search.c +++ b/drivers/pci/search.c @@ -245,8 +245,10 @@ static int match_pci_dev_by_id(struct device *dev, const void *data) { struct pci_dev *pdev = to_pci_dev(dev); const struct pci_device_id *id = data; + struct pci_device_id dev_id; - if (pci_match_one_device(id, pdev)) + dev_id = pci_id_from_device(pdev); + if (pci_match_one_id(id, &dev_id)) return 1; return 0; } @@ -416,9 +418,9 @@ EXPORT_SYMBOL(pci_get_class); * @class: search for a PCI device with this base class code * @from: Previous PCI device found in search, or %NULL for new search. * - * Iterates through the list of known PCI devices. If a PCI device is found + * Iterate through the list of known PCI devices. If a PCI device is found * with a matching base class code, the reference count to the device is - * incremented. See pci_match_one_device() to figure out how does this works. + * incremented. See pci_match_one_id() to figure out how this works. * A new search is initiated by passing %NULL as the @from argument. * Otherwise if @from is not %NULL, searches continue from next device on the * global list. The reference count for @from is always decremented if it is diff --git a/drivers/perf/arm_pmuv3.c b/drivers/perf/arm_pmuv3.c index 8014ff766cff59..b9a8592bf11289 100644 --- a/drivers/perf/arm_pmuv3.c +++ b/drivers/perf/arm_pmuv3.c @@ -1361,7 +1361,7 @@ static int branch_records_alloc(struct arm_pmu *armpmu) struct pmu_hw_events *events_cpu; events_cpu = per_cpu_ptr(armpmu->hw_events, cpu); - events_cpu->branch_stack = kmalloc(size, GFP_KERNEL); + events_cpu->branch_stack = kzalloc(size, GFP_KERNEL); if (!events_cpu->branch_stack) return -ENOMEM; } diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c index dbd0da1116390f..b64cf2313a20cb 100644 --- a/drivers/perf/arm_spe_pmu.c +++ b/drivers/perf/arm_spe_pmu.c @@ -577,7 +577,7 @@ static u64 __arm_spe_pmu_next_off(struct perf_output_handle *handle) * the page boundary following it. Keep the tail boundary if * that's lower. */ - if (handle->wakeup < (handle->head + handle->size) && head <= wakeup) + if ((handle->wakeup - handle->head) < handle->size && head <= wakeup) limit = min(limit, round_up(wakeup, PAGE_SIZE)); if (limit > head) diff --git a/drivers/perf/cxl_pmu.c b/drivers/perf/cxl_pmu.c index 68a54d97d2a8af..39b46550a5109f 100644 --- a/drivers/perf/cxl_pmu.c +++ b/drivers/perf/cxl_pmu.c @@ -635,7 +635,7 @@ static void cxl_pmu_event_start(struct perf_event *event, int flags) cfg = cxl_pmu_config2_get_hdm_decoder(event); else cfg = GENMASK(31, 0); /* No filtering if 0xFFFF_FFFF */ - writeq(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 0)); + writel(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 0)); } cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c index cdcfad2e86b1d3..5b278fd54a161f 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c @@ -2108,7 +2108,7 @@ static const u8 qmp_dp_v4_pre_emphasis_hbr3_hbr2[4][4] = { static const u8 qmp_dp_v4_pre_emphasis_hbr_rbr[4][4] = { { 0x00, 0x0d, 0x14, 0x1a }, { 0x00, 0x0e, 0x15, 0xff }, - { 0x00, 0x0d, 0xff, 0xff }, + { 0x00, 0x0e, 0xff, 0xff }, { 0x03, 0xff, 0xff, 0xff } }; @@ -2325,7 +2325,6 @@ static int qmp_v3_calibrate_dp_phy(struct qmp_combo *qmp); static void qmp_v4_dp_aux_init(struct qmp_combo *qmp); static void qmp_v4_configure_dp_tx(struct qmp_combo *qmp); static int qmp_v4_configure_dp_phy(struct qmp_combo *qmp); -static int qmp_v4_calibrate_dp_phy(struct qmp_combo *qmp); static void qmp_v8_dp_aux_init(struct qmp_combo *qmp); static int qmp_v8_configure_dp_clocks(struct qmp_combo *qmp); @@ -2466,7 +2465,6 @@ static const struct qmp_phy_cfg sar2130p_usb3dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .regs = qmp_v6_usb3phy_regs_layout, .reset_list = msm8996_usb3phy_reset_l, @@ -2603,7 +2601,6 @@ static const struct qmp_phy_cfg sc8180x_usb3dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .reset_list = msm8996_usb3phy_reset_l, .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), @@ -2650,7 +2647,6 @@ static const struct qmp_phy_cfg sc8280xp_usb43dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .reset_list = msm8996_usb3phy_reset_l, .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), @@ -2696,7 +2692,6 @@ static const struct qmp_phy_cfg x1e80100_usb3dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .reset_list = msm8996_usb3phy_reset_l, .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), @@ -2785,7 +2780,6 @@ static const struct qmp_phy_cfg sm8250_usb3dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .reset_list = msm8996_usb3phy_reset_l, .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), @@ -2834,7 +2828,6 @@ static const struct qmp_phy_cfg sm8350_usb3dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .reset_list = msm8996_usb3phy_reset_l, .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), @@ -2882,7 +2875,6 @@ static const struct qmp_phy_cfg sm8550_usb3dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .regs = qmp_v6_usb3phy_regs_layout, .reset_list = msm8996_usb3phy_reset_l, @@ -2928,7 +2920,6 @@ static const struct qmp_phy_cfg sm8650_usb3dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .regs = qmp_v6_usb3phy_regs_layout, .reset_list = msm8996_usb3phy_reset_l, @@ -2974,7 +2965,6 @@ static const struct qmp_phy_cfg sm8750_usb3dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .regs = qmp_v8_usb3phy_regs_layout, .reset_list = msm8996_usb3phy_reset_l, @@ -3020,7 +3010,6 @@ static const struct qmp_phy_cfg glymur_usb3dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v8_configure_dp_clocks, .configure_dp_phy = qmp_v8_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .regs = qmp_v8_n3_usb43dpphy_regs_layout, .reset_list = msm8996_usb3phy_reset_l, @@ -3316,7 +3305,6 @@ static void qmp_v4_dp_aux_init(struct qmp_combo *qmp) writel(0x03, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG7); writel(0xb7, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG8); writel(0x03, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG9); - qmp->dp_aux_cfg = 0; writel(PHY_AUX_STOP_ERR_MASK | PHY_AUX_DEC_ERR_MASK | PHY_AUX_SYNC_ERR_MASK | PHY_AUX_ALIGN_ERR_MASK | @@ -3345,7 +3333,6 @@ static void qmp_v8_dp_aux_init(struct qmp_combo *qmp) writel(0x03, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG7); writel(0xb7, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG8); writel(0x03, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG9); - qmp->dp_aux_cfg = 0; writel(PHY_AUX_STOP_ERR_MASK | PHY_AUX_DEC_ERR_MASK | PHY_AUX_SYNC_ERR_MASK | PHY_AUX_ALIGN_ERR_MASK | @@ -3605,24 +3592,6 @@ static int qmp_v8_configure_dp_phy(struct qmp_combo *qmp) return 0; } -/* - * We need to calibrate the aux setting here as many times - * as the caller tries - */ -static int qmp_v4_calibrate_dp_phy(struct qmp_combo *qmp) -{ - static const u8 cfg1_settings[] = { 0x20, 0x13, 0x23, 0x1d }; - u8 val; - - qmp->dp_aux_cfg++; - qmp->dp_aux_cfg %= ARRAY_SIZE(cfg1_settings); - val = cfg1_settings[qmp->dp_aux_cfg]; - - writel(val, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG1); - - return 0; -} - static int qmp_combo_dp_configure(struct phy *phy, union phy_configure_opts *opts) { const struct phy_configure_opts_dp *dp_opts = &opts->dp; diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c b/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c index d3effad7a074bf..3618812e84d5f0 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c @@ -3554,6 +3554,7 @@ static const struct qmp_pcie_offsets qmp_pcie_offsets_v5 = { .pcs_misc = 0x0600, .tx = 0x0e00, .rx = 0x1000, + .pcs_lane1 = 0x1400, .tx2 = 0x1600, .rx2 = 0x1800, }; diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-usb-legacy.c b/drivers/phy/qualcomm/phy-qcom-qmp-usb-legacy.c index 8bf951b0490cfd..fc490589c8e488 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-usb-legacy.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-usb-legacy.c @@ -542,6 +542,8 @@ struct qmp_usb { enum phy_mode mode; + bool phy_initialized; + struct phy *phy; struct clk_fixed_rate pipe_clk_fixed; @@ -895,6 +897,7 @@ static int qmp_usb_legacy_power_off(struct phy *phy) static int qmp_usb_legacy_enable(struct phy *phy) { + struct qmp_usb *qmp = phy_get_drvdata(phy); int ret; ret = qmp_usb_legacy_init(phy); @@ -904,14 +907,19 @@ static int qmp_usb_legacy_enable(struct phy *phy) ret = qmp_usb_legacy_power_on(phy); if (ret) qmp_usb_legacy_exit(phy); + else + qmp->phy_initialized = true; return ret; } static int qmp_usb_legacy_disable(struct phy *phy) { + struct qmp_usb *qmp = phy_get_drvdata(phy); int ret; + qmp->phy_initialized = false; + ret = qmp_usb_legacy_power_off(phy); if (ret) return ret; @@ -988,7 +996,7 @@ static int __maybe_unused qmp_usb_legacy_runtime_suspend(struct device *dev) dev_vdbg(dev, "Suspending QMP phy, mode:%d\n", qmp->mode); - if (!qmp->phy->init_count) { + if (!qmp->phy_initialized) { dev_vdbg(dev, "PHY not initialized, bailing out\n"); return 0; } @@ -1009,7 +1017,7 @@ static int __maybe_unused qmp_usb_legacy_runtime_resume(struct device *dev) dev_vdbg(dev, "Resuming QMP phy, mode:%d\n", qmp->mode); - if (!qmp->phy->init_count) { + if (!qmp->phy_initialized) { dev_vdbg(dev, "PHY not initialized, bailing out\n"); return 0; } diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-usb.c b/drivers/phy/qualcomm/phy-qcom-qmp-usb.c index c5507168e1354b..bb905c437a01e3 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-usb.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-usb.c @@ -1434,6 +1434,8 @@ struct qmp_usb { enum phy_mode mode; + bool phy_initialized; + struct phy *phy; struct clk_fixed_rate pipe_clk_fixed; @@ -2009,6 +2011,7 @@ static int qmp_usb_power_off(struct phy *phy) static int qmp_usb_enable(struct phy *phy) { + struct qmp_usb *qmp = phy_get_drvdata(phy); int ret; ret = qmp_usb_init(phy); @@ -2018,14 +2021,19 @@ static int qmp_usb_enable(struct phy *phy) ret = qmp_usb_power_on(phy); if (ret) qmp_usb_exit(phy); + else + qmp->phy_initialized = true; return ret; } static int qmp_usb_disable(struct phy *phy) { + struct qmp_usb *qmp = phy_get_drvdata(phy); int ret; + qmp->phy_initialized = false; + ret = qmp_usb_power_off(phy); if (ret) return ret; @@ -2101,7 +2109,7 @@ static int __maybe_unused qmp_usb_runtime_suspend(struct device *dev) dev_vdbg(dev, "Suspending QMP phy, mode:%d\n", qmp->mode); - if (!qmp->phy->init_count) { + if (!qmp->phy_initialized) { dev_vdbg(dev, "PHY not initialized, bailing out\n"); return 0; } @@ -2121,7 +2129,7 @@ static int __maybe_unused qmp_usb_runtime_resume(struct device *dev) dev_vdbg(dev, "Resuming QMP phy, mode:%d\n", qmp->mode); - if (!qmp->phy->init_count) { + if (!qmp->phy_initialized) { dev_vdbg(dev, "PHY not initialized, bailing out\n"); return 0; } diff --git a/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c b/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c index f48faa2929a682..7137200c3f310a 100644 --- a/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c +++ b/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "phy-qcom-qmp-pcs-sgmii.h" #include "phy-qcom-qmp-qserdes-com-v5.h" @@ -26,7 +27,15 @@ #define QSERDES_PCS_SGMIIPHY_READY BIT(7) #define QSERDES_COM_C_PLL_LOCKED BIT(1) +static const struct regulator_bulk_data qcom_dwmac_sgmii_phy_vregs[] = { + { .supply = "vdda-0p9", .init_load_uA = 46000 }, + { .supply = "vdda-1p2", .init_load_uA = 15000 }, +}; + +#define QCOM_SGMII_NUM_SUPPLIES ARRAY_SIZE(qcom_dwmac_sgmii_phy_vregs) + struct qcom_dwmac_sgmii_phy_data { + struct regulator_bulk_data *vregs; struct regmap *regmap; struct clk *refclk; phy_interface_t interface; @@ -273,13 +282,24 @@ static int qcom_dwmac_sgmii_phy_power_on(struct phy *phy) struct qcom_dwmac_sgmii_phy_data *data = phy_get_drvdata(phy); int ret; - ret = clk_prepare_enable(data->refclk); - if (ret < 0) + ret = regulator_bulk_enable(QCOM_SGMII_NUM_SUPPLIES, data->vregs); + if (ret) return ret; + ret = clk_prepare_enable(data->refclk); + if (ret) + goto err_disable_regulators; + ret = qcom_dwmac_sgmii_phy_calibrate(phy); - if (ret < 0) - clk_disable_unprepare(data->refclk); + if (ret) + goto err_disable_clk; + + return 0; + +err_disable_clk: + clk_disable_unprepare(data->refclk); +err_disable_regulators: + regulator_bulk_disable(QCOM_SGMII_NUM_SUPPLIES, data->vregs); return ret; } @@ -296,6 +316,8 @@ static int qcom_dwmac_sgmii_phy_power_off(struct phy *phy) clk_disable_unprepare(data->refclk); + regulator_bulk_disable(QCOM_SGMII_NUM_SUPPLIES, data->vregs); + return 0; } @@ -357,6 +379,7 @@ static int qcom_dwmac_sgmii_phy_probe(struct platform_device *pdev) struct phy_provider *provider; void __iomem *base; struct phy *phy; + int ret; data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); if (!data) @@ -381,6 +404,12 @@ static int qcom_dwmac_sgmii_phy_probe(struct platform_device *pdev) if (IS_ERR(data->refclk)) return PTR_ERR(data->refclk); + ret = devm_regulator_bulk_get_const(dev, QCOM_SGMII_NUM_SUPPLIES, + qcom_dwmac_sgmii_phy_vregs, + &data->vregs); + if (ret) + return ret; + provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); if (IS_ERR(provider)) return PTR_ERR(provider); diff --git a/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c b/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c index eb0b0f61d98e03..980ad1fb1e2e3c 100644 --- a/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c +++ b/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c @@ -599,8 +599,18 @@ static int qcom_snps_hsphy_probe(struct platform_device *pdev) return dev_err_probe(dev, ret, "failed to get regulator supplies\n"); + /* + * Enable runtime PM before creating the PHY, phy_create() only enables + * it on the PHY device if already enabled on the parent. Hold a usage + * reference so callbacks cannot run before the PHY is ready. + */ + pm_runtime_get_noresume(dev); pm_runtime_set_active(dev); - pm_runtime_enable(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) { + pm_runtime_put_noidle(dev); + return ret; + } /* * Prevent runtime pm from being ON by default. Users can enable * it using power/control in sysfs. @@ -611,6 +621,7 @@ static int qcom_snps_hsphy_probe(struct platform_device *pdev) if (IS_ERR(generic_phy)) { ret = PTR_ERR(generic_phy); dev_err(dev, "failed to create phy, %d\n", ret); + pm_runtime_put_noidle(dev); return ret; } hsphy->phy = generic_phy; @@ -620,12 +631,15 @@ static int qcom_snps_hsphy_probe(struct platform_device *pdev) qcom_snps_hsphy_read_override_param_seq(dev); phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); - if (!IS_ERR(phy_provider)) - dev_dbg(dev, "Registered Qcom-SNPS HS phy\n"); - else - pm_runtime_disable(dev); + if (IS_ERR(phy_provider)) { + pm_runtime_put_noidle(dev); + return PTR_ERR(phy_provider); + } - return PTR_ERR_OR_ZERO(phy_provider); + dev_dbg(dev, "Registered Qcom-SNPS HS phy\n"); + pm_runtime_put(dev); + + return 0; } static struct platform_driver qcom_snps_hsphy_driver = { diff --git a/drivers/phy/renesas/phy-rcar-gen2.c b/drivers/phy/renesas/phy-rcar-gen2.c index 6c671254c62580..5a272e25e051c2 100644 --- a/drivers/phy/renesas/phy-rcar-gen2.c +++ b/drivers/phy/renesas/phy-rcar-gen2.c @@ -405,7 +405,6 @@ static int rcar_gen2_phy_probe(struct platform_device *pdev) data->gen2_phy_ops); if (IS_ERR(phy->phy)) { dev_err(dev, "Failed to create PHY\n"); - of_node_put(np); return PTR_ERR(phy->phy); } phy_set_drvdata(phy->phy, phy); diff --git a/drivers/phy/renesas/phy-rcar-gen3-usb2.c b/drivers/phy/renesas/phy-rcar-gen3-usb2.c index 9a45d840efeb25..7dab11bfcb1cd1 100644 --- a/drivers/phy/renesas/phy-rcar-gen3-usb2.c +++ b/drivers/phy/renesas/phy-rcar-gen3-usb2.c @@ -898,18 +898,29 @@ static int rcar_gen3_phy_usb2_vbus_regulator_get_exclusive_enable(struct rcar_ge int ret; channel->vbus = devm_regulator_get_exclusive(dev, "vbus"); - if (IS_ERR(channel->vbus)) - return PTR_ERR(channel->vbus); + if (IS_ERR(channel->vbus)) { + ret = PTR_ERR(channel->vbus); + /* If vbus-regulator node was present vbus regulator should be available */ + if (channel->otg_internal_reg) + return ret; + + if (ret == -EPROBE_DEFER) + return ret; - if (!enable) return 0; + } - ret = regulator_enable(channel->vbus); - if (ret) - return ret; + if (enable) { + ret = regulator_enable(channel->vbus); + if (ret) + return ret; + } + + if (regulator_is_enabled(channel->vbus)) + return devm_add_action_or_reset(dev, rcar_gen3_phy_usb2_vbus_disable_action, + channel->vbus); - return devm_add_action_or_reset(dev, rcar_gen3_phy_usb2_vbus_disable_action, - channel->vbus); + return 0; } static int rcar_gen3_phy_usb2_vbus_regulator_register(struct rcar_gen3_chan *channel) diff --git a/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c b/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c index c79fb53d8ee5c5..5281f8dea0ad3d 100644 --- a/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c +++ b/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c @@ -170,7 +170,7 @@ static const struct hsfreq_range rk1808_mipidphy_hsfreq_ranges[] = { { 299, 0x06}, { 399, 0x08}, { 499, 0x0b}, { 599, 0x0e}, { 699, 0x10}, { 799, 0x12}, { 999, 0x16}, {1199, 0x1e}, {1399, 0x23}, {1599, 0x2d}, {1799, 0x32}, {1999, 0x37}, - {2199, 0x3c}, {2399, 0x41}, {2499, 0x46} + {2199, 0x3c}, {2399, 0x41}, {2500, 0x46} }; static const struct hsfreq_range rk3326_mipidphy_hsfreq_ranges[] = { diff --git a/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c b/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c index 2d973bc37f076f..c24e7f2841703e 100644 --- a/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c +++ b/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c @@ -413,6 +413,7 @@ struct rk_hdptx_phy { /* clk provider */ struct clk_hw hw; + bool pll_config_dirty; bool restrict_rate_change; atomic_t usage_count; @@ -1260,13 +1261,19 @@ static int rk_hdptx_tmds_ropll_cmn_config(struct rk_hdptx_phy *hdptx) static int rk_hdptx_pll_cmn_config(struct rk_hdptx_phy *hdptx) { + int ret; + if (hdptx->hdmi_cfg.rate <= HDMI20_MAX_RATE) - return rk_hdptx_tmds_ropll_cmn_config(hdptx); + ret = rk_hdptx_tmds_ropll_cmn_config(hdptx); + else if (hdptx->hdmi_cfg.rate == FRL_8G4L_RATE) + ret = rk_hdptx_frl_lcpll_ropll_cmn_config(hdptx); + else + ret = rk_hdptx_frl_lcpll_cmn_config(hdptx); - if (hdptx->hdmi_cfg.rate == FRL_8G4L_RATE) - return rk_hdptx_frl_lcpll_ropll_cmn_config(hdptx); + if (!ret) + hdptx->pll_config_dirty = false; - return rk_hdptx_frl_lcpll_cmn_config(hdptx); + return ret; } static int rk_hdptx_frl_lcpll_mode_config(struct rk_hdptx_phy *hdptx) @@ -1347,25 +1354,22 @@ static int rk_hdptx_phy_consumer_get(struct rk_hdptx_phy *hdptx) return 0; ret = regmap_read(hdptx->grf, GRF_HDPTX_STATUS, &status); - if (ret) - goto dec_usage; - - if (status & HDPTX_O_PLL_LOCK_DONE) - dev_warn(hdptx->dev, "PLL locked by unknown consumer!\n"); + if (ret) { + atomic_dec(&hdptx->usage_count); + return ret; + } if (mode == PHY_MODE_DP) { rk_hdptx_dp_reset(hdptx); } else { - ret = rk_hdptx_pll_cmn_config(hdptx); - if (ret) - goto dec_usage; + /* + * Ignore PLL config errors at this point as pll_config_dirty + * was not reset and, therefore, operation will be retried. + */ + rk_hdptx_pll_cmn_config(hdptx); } return 0; - -dec_usage: - atomic_dec(&hdptx->usage_count); - return ret; } static int rk_hdptx_phy_consumer_put(struct rk_hdptx_phy *hdptx, bool force) @@ -1700,13 +1704,18 @@ static int rk_hdptx_phy_power_on(struct phy *phy) if (ret) rk_hdptx_phy_consumer_put(hdptx, true); } else { - regmap_write(hdptx->grf, GRF_HDPTX_CON0, - HDPTX_MODE_SEL << 16 | FIELD_PREP(HDPTX_MODE_SEL, 0x0)); + if (hdptx->pll_config_dirty) + ret = rk_hdptx_pll_cmn_config(hdptx); - if (hdptx->hdmi_cfg.mode == PHY_HDMI_MODE_FRL) - ret = rk_hdptx_frl_lcpll_mode_config(hdptx); - else - ret = rk_hdptx_tmds_ropll_mode_config(hdptx); + if (!ret) { + regmap_write(hdptx->grf, GRF_HDPTX_CON0, + HDPTX_MODE_SEL << 16 | FIELD_PREP(HDPTX_MODE_SEL, 0x0)); + + if (hdptx->hdmi_cfg.mode == PHY_HDMI_MODE_FRL) + ret = rk_hdptx_frl_lcpll_mode_config(hdptx); + else + ret = rk_hdptx_tmds_ropll_mode_config(hdptx); + } if (ret) rk_hdptx_phy_consumer_put(hdptx, true); @@ -2081,7 +2090,10 @@ static int rk_hdptx_phy_configure(struct phy *phy, union phy_configure_opts *opt dev_err(hdptx->dev, "invalid hdmi params for phy configure\n"); } else { hdptx->restrict_rate_change = true; - dev_dbg(hdptx->dev, "%s rate=%llu bpc=%u\n", __func__, + hdptx->pll_config_dirty = true; + + dev_dbg(hdptx->dev, "%s %s rate=%llu bpc=%u\n", __func__, + hdptx->hdmi_cfg.mode ? "FRL" : "TMDS", hdptx->hdmi_cfg.rate, hdptx->hdmi_cfg.bpc); } @@ -2168,7 +2180,7 @@ static u64 rk_hdptx_phy_clk_calc_rate_from_pll_cfg(struct rk_hdptx_phy *hdptx) struct lcpll_config lcpll_hw; struct ropll_config ropll_hw; u64 fout, sdm; - u32 mode, val; + u32 mode, bpc, val; int ret, i; ret = regmap_read(hdptx->regmap, CMN_REG(0008), &mode); @@ -2206,16 +2218,31 @@ static u64 rk_hdptx_phy_clk_calc_rate_from_pll_cfg(struct rk_hdptx_phy *hdptx) return 0; lcpll_hw.sdc_n = (val & LCPLL_SDC_N_MASK) >> 1; - for (i = 0; i < ARRAY_SIZE(rk_hdptx_frl_lcpll_cfg); i++) { - const struct lcpll_config *cfg = &rk_hdptx_frl_lcpll_cfg[i]; + ret = regmap_read(hdptx->grf, GRF_HDPTX_CON0, &val); + if (ret) + return 0; - if (cfg->pms_mdiv == lcpll_hw.pms_mdiv && - cfg->pms_sdiv == lcpll_hw.pms_sdiv && - cfg->sdm_num_sign == lcpll_hw.sdm_num_sign && - cfg->sdm_num == lcpll_hw.sdm_num && - cfg->sdm_deno == lcpll_hw.sdm_deno && - cfg->sdc_n == lcpll_hw.sdc_n) - return cfg->rate; + if (val & LC_REF_CLK_SEL) { + if (lcpll_hw.pms_mdiv == 0x6b && + lcpll_hw.sdm_num_sign == 0x01 && + lcpll_hw.sdm_num == 0x02 && + lcpll_hw.sdm_deno == 0x09 && + lcpll_hw.sdc_n == FIELD_GET(LCPLL_SDC_N_MASK, 0x02)) + return FRL_8G4L_RATE; + } else { + const struct lcpll_config *cfg; + + for (i = 0; i < ARRAY_SIZE(rk_hdptx_frl_lcpll_cfg); i++) { + cfg = &rk_hdptx_frl_lcpll_cfg[i]; + + if (cfg->pms_mdiv == lcpll_hw.pms_mdiv && + cfg->pms_sdiv == lcpll_hw.pms_sdiv && + cfg->sdm_num_sign == lcpll_hw.sdm_num_sign && + cfg->sdm_num == lcpll_hw.sdm_num && + cfg->sdm_deno == lcpll_hw.sdm_deno && + cfg->sdc_n == lcpll_hw.sdc_n) + return cfg->rate; + } } dev_dbg(hdptx->dev, "%s no FRL match found\n", __func__); @@ -2266,21 +2293,42 @@ static u64 rk_hdptx_phy_clk_calc_rate_from_pll_cfg(struct rk_hdptx_phy *hdptx) if (ret) return 0; ropll_hw.pms_sdiv = ((val & PLL_PCG_POSTDIV_SEL_MASK) >> 4) + 1; + bpc = (FIELD_GET(PLL_PCG_CLK_SEL_MASK, val) << 1) + 8; fout = PLL_REF_CLK * ropll_hw.pms_mdiv; if (ropll_hw.sdm_en) { + val = 16U * ropll_hw.sdm_deno * + (ropll_hw.sdc_deno * ropll_hw.sdc_n - ropll_hw.sdc_num); + if (!val) { + /* + * The PLL config currently stored in hardware can't be + * translated into a rate. The next .set_rate() should + * program a valid configuration and help with recovery. + */ + dev_dbg(hdptx->dev, "Invalid ROPLL hw state: deno == 0\n"); + return 0; + } + sdm = div_u64(PLL_REF_CLK * ropll_hw.sdc_deno * - ropll_hw.pms_mdiv * ropll_hw.sdm_num, - 16 * ropll_hw.sdm_deno * - (ropll_hw.sdc_deno * ropll_hw.sdc_n - ropll_hw.sdc_num)); + ropll_hw.pms_mdiv * ropll_hw.sdm_num, val); + + if (ropll_hw.sdm_num_sign) { + if (sdm > fout) { + /* + * Similarly to the case above, it is expected + * the next .set_rate() will help with recovery. + */ + dev_dbg(hdptx->dev, "Invalid ROPLL hw state: sdm > fout\n"); + return 0; + } - if (ropll_hw.sdm_num_sign) fout = fout - sdm; - else + } else { fout = fout + sdm; + } } - return div_u64(fout * 2, ropll_hw.pms_sdiv * 10); + return DIV_ROUND_CLOSEST_ULL(fout * 2 * 8, ropll_hw.pms_sdiv * 10 * bpc); } static unsigned long rk_hdptx_phy_clk_recalc_rate(struct clk_hw *hw, @@ -2288,19 +2336,13 @@ static unsigned long rk_hdptx_phy_clk_recalc_rate(struct clk_hw *hw, { struct rk_hdptx_phy *hdptx = to_rk_hdptx_phy(hw); u32 status; - u64 rate; int ret; ret = regmap_read(hdptx->grf, GRF_HDPTX_CON0, &status); if (ret || !(status & HDPTX_I_PLL_EN)) return 0; - rate = rk_hdptx_phy_clk_calc_rate_from_pll_cfg(hdptx); - - if (hdptx->hdmi_cfg.mode == PHY_HDMI_MODE_FRL) - return rate; - - return DIV_ROUND_CLOSEST_ULL(rate * 8, hdptx->hdmi_cfg.bpc); + return rk_hdptx_phy_clk_calc_rate_from_pll_cfg(hdptx); } static int rk_hdptx_phy_clk_determine_rate(struct clk_hw *hw, @@ -2308,8 +2350,19 @@ static int rk_hdptx_phy_clk_determine_rate(struct clk_hw *hw, { struct rk_hdptx_phy *hdptx = to_rk_hdptx_phy(hw); - if (hdptx->hdmi_cfg.mode == PHY_HDMI_MODE_FRL) - return hdptx->hdmi_cfg.rate; + /* + * Invalidate current clock rate to ensure rk_hdptx_phy_clk_set_rate() + * will be invoked to commit PLL configuration. + */ + if (hdptx->pll_config_dirty) { + req->rate = 0; + return 0; + } + + if (hdptx->hdmi_cfg.mode == PHY_HDMI_MODE_FRL) { + req->rate = hdptx->hdmi_cfg.rate; + return 0; + } /* * FIXME: Temporarily allow altering TMDS char rate via CCF. @@ -2341,17 +2394,6 @@ static int rk_hdptx_phy_clk_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) { struct rk_hdptx_phy *hdptx = to_rk_hdptx_phy(hw); - unsigned long long link_rate = rate; - - if (hdptx->hdmi_cfg.mode != PHY_HDMI_MODE_FRL) - link_rate = DIV_ROUND_CLOSEST_ULL(rate * hdptx->hdmi_cfg.bpc, 8); - - /* Revert any unlikely link rate change since determine_rate() */ - if (hdptx->hdmi_cfg.rate != link_rate) { - dev_warn(hdptx->dev, "Reverting unexpected rate change from %llu to %llu\n", - link_rate, hdptx->hdmi_cfg.rate); - hdptx->hdmi_cfg.rate = link_rate; - } /* * The link rate would be normally programmed in HW during diff --git a/drivers/phy/starfive/phy-jh7110-dphy-rx.c b/drivers/phy/starfive/phy-jh7110-dphy-rx.c index 0b039e1f71c55f..d06f21ad633251 100644 --- a/drivers/phy/starfive/phy-jh7110-dphy-rx.c +++ b/drivers/phy/starfive/phy-jh7110-dphy-rx.c @@ -150,6 +150,7 @@ static int stf_dphy_probe(struct platform_device *pdev) { struct phy_provider *phy_provider; struct stf_dphy *dphy; + int ret; dphy = devm_kzalloc(&pdev->dev, sizeof(*dphy), GFP_KERNEL); if (!dphy) @@ -190,7 +191,9 @@ static int stf_dphy_probe(struct platform_device *pdev) return PTR_ERR(dphy->phy); } - pm_runtime_enable(&pdev->dev); + ret = devm_pm_runtime_enable(&pdev->dev); + if (ret) + return ret; phy_set_drvdata(dphy->phy, dphy); phy_provider = devm_of_phy_provider_register(&pdev->dev, diff --git a/drivers/phy/starfive/phy-jh7110-dphy-tx.c b/drivers/phy/starfive/phy-jh7110-dphy-tx.c index c64d1c91b1307e..181491a9380790 100644 --- a/drivers/phy/starfive/phy-jh7110-dphy-tx.c +++ b/drivers/phy/starfive/phy-jh7110-dphy-tx.c @@ -392,6 +392,7 @@ static int stf_dphy_probe(struct platform_device *pdev) { struct phy_provider *phy_provider; struct stf_dphy *dphy; + int ret; dphy = devm_kzalloc(&pdev->dev, sizeof(*dphy), GFP_KERNEL); if (!dphy) @@ -406,7 +407,9 @@ static int stf_dphy_probe(struct platform_device *pdev) if (IS_ERR(dphy->topsys)) return PTR_ERR(dphy->topsys); - pm_runtime_enable(&pdev->dev); + ret = devm_pm_runtime_enable(&pdev->dev); + if (ret) + return ret; dphy->txesc_clk = devm_clk_get(&pdev->dev, "txesc"); if (IS_ERR(dphy->txesc_clk)) diff --git a/drivers/phy/sunplus/phy-sunplus-usb2.c b/drivers/phy/sunplus/phy-sunplus-usb2.c index 637a5fbae6d9a9..0ad4c7160d17c2 100644 --- a/drivers/phy/sunplus/phy-sunplus-usb2.c +++ b/drivers/phy/sunplus/phy-sunplus-usb2.c @@ -116,11 +116,11 @@ static int sp_uphy_init(struct phy *phy) ret = clk_prepare_enable(usbphy->phy_clk); if (ret) - goto err_clk; + return ret; ret = reset_control_deassert(usbphy->rstc); if (ret) - goto err_reset; + goto err_clk; /* Default value modification */ writel(HIGH_MASK_BITS | 0x4002, usbphy->moon4_regs + UPHY_CONTROL0); @@ -129,7 +129,7 @@ static int sp_uphy_init(struct phy *phy) /* disconnect voltage */ ret = update_disc_vol(usbphy); if (ret < 0) - return ret; + goto err_reset; /* board uphy 0 internal register modification for tid certification */ val = readl(usbphy->phy_regs + CONFIG9); diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig index eda54aa5fde6fc..bf73bc709c1811 100644 --- a/drivers/pinctrl/Kconfig +++ b/drivers/pinctrl/Kconfig @@ -275,8 +275,8 @@ config PINCTRL_GEMINI config PINCTRL_GENERIC_MUX tristate "Generic Pinctrl driver by using multiplexer" - depends on MULTIPLEXER depends on OF + select MULTIPLEXER select PINMUX select GENERIC_PINCTRL help @@ -488,6 +488,7 @@ config PINCTRL_PIC32 depends on MACH_PIC32 || COMPILE_TEST select PINMUX select GENERIC_PINCONF + select GPIOLIB select GPIOLIB_IRQCHIP help This is the pin controller and gpio driver for Microchip PIC32 @@ -564,6 +565,7 @@ config PINCTRL_STMFX depends on I2C depends on HAS_IOMEM select GENERIC_PINCONF + select GPIOLIB select GPIOLIB_IRQCHIP select MFD_STMFX help diff --git a/drivers/pinctrl/airoha/pinctrl-airoha.c b/drivers/pinctrl/airoha/pinctrl-airoha.c index 04b4424c688b2c..6a40a776d31582 100644 --- a/drivers/pinctrl/airoha/pinctrl-airoha.c +++ b/drivers/pinctrl/airoha/pinctrl-airoha.c @@ -49,7 +49,7 @@ /* MUX */ #define REG_GPIO_2ND_I2C_MODE 0x0214 -#define GPIO_MDC_IO_MASTER_MODE_MODE BIT(14) +#define GPIO_MDC_IO_MASTER_MODE_MASK BIT(14) #define GPIO_I2C_MASTER_MODE_MODE BIT(13) #define GPIO_I2S_MODE_MASK BIT(12) #define GPIO_I2C_SLAVE_MODE_MODE BIT(11) @@ -66,6 +66,7 @@ #define GPIO_2ND_I2C_MODE_MASK BIT(0) #define REG_GPIO_SPI_CS1_MODE 0x0218 +#define AN7583_GPIO_MDC_IO_MASTER_MODE_MASK BIT(22) #define GPIO_PCM_SPI_CS4_MODE_MASK BIT(21) #define GPIO_PCM_SPI_CS3_MODE_MASK BIT(20) #define GPIO_PCM_SPI_CS2_MODE_P156_MASK BIT(19) @@ -84,6 +85,18 @@ #define GPIO_SPI_CS1_MODE_MASK BIT(0) #define REG_GPIO_PON_MODE 0x021c +#define AN7583_MDIO_0_GPIO_MODE_MASK BIT(26) +#define AN7583_MDC_0_GPIO_MODE_MASK BIT(25) +#define AN7583_UART_RXD_GPIO_MODE_MASK BIT(24) +#define AN7583_UART_TXD_GPIO_MODE_MASK BIT(23) +#define AN7583_SPI_MISO_GPIO_MODE_MASK BIT(22) +#define AN7583_SPI_MOSI_GPIO_MODE_MASK BIT(21) +#define AN7583_SPI_CS_GPIO_MODE_MASK BIT(20) +#define AN7583_SPI_CLK_GPIO_MODE_MASK BIT(19) +#define AN7583_I2C1_SDA_GPIO_MODE_MASK BIT(18) +#define AN7583_I2C1_SCL_GPIO_MODE_MASK BIT(17) +#define AN7583_I2C0_SDA_GPIO_MODE_MASK BIT(16) +#define AN7583_I2C0_SCL_GPIO_MODE_MASK BIT(15) #define GPIO_PARALLEL_NAND_MODE_MASK BIT(14) #define GPIO_SGMII_MDIO_MODE_MASK BIT(13) #define GPIO_PCIE_RESET2_MASK BIT(12) @@ -184,8 +197,8 @@ #define I2C_SDA_PU_MASK BIT(0) #define REG_I2C_SDA_PD 0x0048 -#define AN7583_I2C1_SDA_PD_MASK BIT(16) -#define AN7583_I2C1_SCL_PD_MASK BIT(15) +#define AN7583_I2C1_SCL_PD_MASK BIT(16) +#define AN7583_I2C1_SDA_PD_MASK BIT(15) #define SPI_MISO_PD_MASK BIT(14) #define SPI_MOSI_PD_MASK BIT(13) #define SPI_CLK_PD_MASK BIT(12) @@ -469,9 +482,9 @@ static struct pinctrl_pin_desc en7581_pinctrl_pins[] = { PINCTRL_PIN(57, "gpio44"), PINCTRL_PIN(58, "gpio45"), PINCTRL_PIN(59, "gpio46"), - PINCTRL_PIN(61, "pcie_reset0"), - PINCTRL_PIN(62, "pcie_reset1"), - PINCTRL_PIN(63, "pcie_reset2"), + PINCTRL_PIN(60, "pcie_reset0"), + PINCTRL_PIN(61, "pcie_reset1"), + PINCTRL_PIN(62, "pcie_reset2"), }; static const int en7581_pon_pins[] = { 49, 50, 51, 52, 53, 54 }; @@ -554,9 +567,12 @@ static const int en7581_gpio43_pins[] = { 56 }; static const int en7581_gpio44_pins[] = { 57 }; static const int en7581_gpio45_pins[] = { 58 }; static const int en7581_gpio46_pins[] = { 59 }; -static const int en7581_pcie_reset0_pins[] = { 61 }; -static const int en7581_pcie_reset1_pins[] = { 62 }; -static const int en7581_pcie_reset2_pins[] = { 63 }; +static const int en7581_gpio47_pins[] = { 60 }; +static const int en7581_gpio48_pins[] = { 61 }; +static const int en7581_gpio49_pins[] = { 62 }; +static const int en7581_pcie_reset0_pins[] = { 60 }; +static const int en7581_pcie_reset1_pins[] = { 61 }; +static const int en7581_pcie_reset2_pins[] = { 62 }; static const struct pingroup en7581_pinctrl_groups[] = { PINCTRL_PIN_GROUP("pon", en7581_pon), @@ -639,6 +655,9 @@ static const struct pingroup en7581_pinctrl_groups[] = { PINCTRL_PIN_GROUP("gpio44", en7581_gpio44), PINCTRL_PIN_GROUP("gpio45", en7581_gpio45), PINCTRL_PIN_GROUP("gpio46", en7581_gpio46), + PINCTRL_PIN_GROUP("gpio47", en7581_gpio47), + PINCTRL_PIN_GROUP("gpio48", en7581_gpio48), + PINCTRL_PIN_GROUP("gpio49", en7581_gpio49), PINCTRL_PIN_GROUP("pcie_reset0", en7581_pcie_reset0), PINCTRL_PIN_GROUP("pcie_reset1", en7581_pcie_reset1), PINCTRL_PIN_GROUP("pcie_reset2", en7581_pcie_reset2), @@ -719,7 +738,7 @@ static const int an7583_jtag_udi_pins[] = { 23, 24, 22, 25, 26 }; static const int an7583_jtag_dfd_pins[] = { 23, 24, 22, 25, 26 }; static const int an7583_pcm1_pins[] = { 10, 11, 12, 13, 14 }; static const int an7583_pcm2_pins[] = { 28, 29, 30, 31, 24 }; -static const int an7583_spi_pins[] = { 28, 29, 30, 31 }; +static const int an7583_spi_pins[] = { 45, 46, 47, 48 }; static const int an7583_spi_quad_pins[] = { 25, 26 }; static const int an7583_spi_cs1_pins[] = { 27 }; static const int an7583_pcm_spi_pins[] = { 28, 29, 30, 31, 10, 11, 12, 13 }; @@ -776,6 +795,10 @@ static const int an7583_gpio45_pins[] = { 47 }; static const int an7583_gpio46_pins[] = { 48 }; static const int an7583_gpio47_pins[] = { 49 }; static const int an7583_gpio48_pins[] = { 50 }; +static const int an7583_gpio49_pins[] = { 51 }; +static const int an7583_gpio50_pins[] = { 52 }; +static const int an7583_gpio51_pins[] = { 53 }; +static const int an7583_gpio52_pins[] = { 54 }; static const int an7583_pcie_reset0_pins[] = { 51 }; static const int an7583_pcie_reset1_pins[] = { 52 }; @@ -856,6 +879,10 @@ static const struct pingroup an7583_pinctrl_groups[] = { PINCTRL_PIN_GROUP("gpio46", an7583_gpio46), PINCTRL_PIN_GROUP("gpio47", an7583_gpio47), PINCTRL_PIN_GROUP("gpio48", an7583_gpio48), + PINCTRL_PIN_GROUP("gpio49", an7583_gpio49), + PINCTRL_PIN_GROUP("gpio50", an7583_gpio50), + PINCTRL_PIN_GROUP("gpio51", an7583_gpio51), + PINCTRL_PIN_GROUP("gpio52", an7583_gpio52), PINCTRL_PIN_GROUP("pcie_reset0", an7583_pcie_reset0), PINCTRL_PIN_GROUP("pcie_reset1", an7583_pcie_reset1), }; @@ -882,8 +909,14 @@ static const char *const an7583_pcm_spi_groups[] = { "pcm_spi", static const char *const i2s_groups[] = { "i2s" }; static const char *const emmc_groups[] = { "emmc" }; static const char *const pnand_groups[] = { "pnand" }; +static const char *const gpio_groups[] = { "gpio47", "gpio48", "gpio49" }; static const char *const pcie_reset_groups[] = { "pcie_reset0", "pcie_reset1", "pcie_reset2" }; +static const char *const an7583_gpio_groups[] = { "gpio39", "gpio40", "gpio41", + "gpio42", "gpio43", "gpio44", + "gpio45", "gpio46", "gpio47", + "gpio48", "gpio49", "gpio50", + "gpio51", "gpio52" }; static const char *const an7583_pcie_reset_groups[] = { "pcie_reset0", "pcie_reset1" }; static const char *const pwm_groups[] = { "gpio0", "gpio1", "gpio2", "gpio3", @@ -906,7 +939,8 @@ static const char *const pwm_groups[] = { "gpio0", "gpio1", "gpio40", "gpio41", "gpio42", "gpio43", "gpio44", "gpio45", - "gpio46" }; + "gpio46", "gpio47", + "gpio48", "gpio49" }; static const char *const an7583_pwm_groups[] = { "gpio0", "gpio1", "gpio2", "gpio3", "gpio4", "gpio5", @@ -929,7 +963,8 @@ static const char *const an7583_pwm_groups[] = { "gpio0", "gpio1", "gpio42", "gpio43", "gpio44", "gpio45", "gpio46", "gpio47", - "gpio48" }; + "gpio48", "gpio49", + "gpio50", "gpio51" }; static const char *const phy1_led0_groups[] = { "gpio33", "gpio34", "gpio35", "gpio42" }; static const char *const phy2_led0_groups[] = { "gpio33", "gpio34", @@ -1026,8 +1061,8 @@ static const struct airoha_pinctrl_func_group mdio_func_group[] = { .regmap[0] = { AIROHA_FUNC_MUX, REG_GPIO_2ND_I2C_MODE, - GPIO_MDC_IO_MASTER_MODE_MODE, - GPIO_MDC_IO_MASTER_MODE_MODE + GPIO_MDC_IO_MASTER_MODE_MASK, + GPIO_MDC_IO_MASTER_MODE_MASK }, .regmap[1] = { AIROHA_FUNC_MUX, @@ -1051,8 +1086,8 @@ static const struct airoha_pinctrl_func_group an7583_mdio_func_group[] = { .regmap[1] = { AIROHA_FUNC_MUX, REG_GPIO_SPI_CS1_MODE, - GPIO_MDC_IO_MASTER_MODE_MODE, - GPIO_MDC_IO_MASTER_MODE_MODE + AN7583_GPIO_MDC_IO_MASTER_MODE_MASK, + AN7583_GPIO_MDC_IO_MASTER_MODE_MASK }, .regmap_size = 2, }, @@ -1405,6 +1440,45 @@ static const struct airoha_pinctrl_func_group pnand_func_group[] = { }, }; +#define AIROHA_PINCTRL_GPIO(gpio, mux_val) \ + { \ + .name = (gpio), \ + .regmap[0] = { \ + AIROHA_FUNC_MUX, \ + REG_GPIO_PON_MODE, \ + (mux_val), \ + (mux_val) \ + }, \ + .regmap_size = 1, \ + } + +#define AIROHA_PINCTRL_GPIO_EXT(gpio, mux_val, smux_val) \ + { \ + .name = (gpio), \ + .regmap[0] = { \ + AIROHA_FUNC_PWM_EXT_MUX, \ + REG_GPIO_FLASH_MODE_CFG_EXT, \ + (mux_val), \ + 0 \ + }, \ + .regmap[1] = { \ + AIROHA_FUNC_MUX, \ + REG_GPIO_PON_MODE, \ + (smux_val), \ + (smux_val) \ + }, \ + .regmap_size = 2, \ + } + +static const struct airoha_pinctrl_func_group gpio_func_group[] = { + AIROHA_PINCTRL_GPIO_EXT("gpio47", GPIO47_FLASH_MODE_CFG, + GPIO_PCIE_RESET0_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio48", GPIO48_FLASH_MODE_CFG, + GPIO_PCIE_RESET1_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio49", GPIO49_FLASH_MODE_CFG, + GPIO_PCIE_RESET2_MASK), +}; + static const struct airoha_pinctrl_func_group pcie_reset_func_group[] = { { .name = "pcie_reset0", @@ -1412,7 +1486,7 @@ static const struct airoha_pinctrl_func_group pcie_reset_func_group[] = { AIROHA_FUNC_MUX, REG_GPIO_PON_MODE, GPIO_PCIE_RESET0_MASK, - GPIO_PCIE_RESET0_MASK + 0 }, .regmap_size = 1, }, { @@ -1421,7 +1495,7 @@ static const struct airoha_pinctrl_func_group pcie_reset_func_group[] = { AIROHA_FUNC_MUX, REG_GPIO_PON_MODE, GPIO_PCIE_RESET1_MASK, - GPIO_PCIE_RESET1_MASK + 0 }, .regmap_size = 1, }, { @@ -1430,12 +1504,42 @@ static const struct airoha_pinctrl_func_group pcie_reset_func_group[] = { AIROHA_FUNC_MUX, REG_GPIO_PON_MODE, GPIO_PCIE_RESET2_MASK, - GPIO_PCIE_RESET2_MASK + 0 }, .regmap_size = 1, }, }; +static const struct airoha_pinctrl_func_group an7583_gpio_func_group[] = { + AIROHA_PINCTRL_GPIO_EXT("gpio39", GPIO39_FLASH_MODE_CFG, + AN7583_I2C0_SCL_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio40", GPIO40_FLASH_MODE_CFG, + AN7583_I2C0_SDA_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio41", GPIO41_FLASH_MODE_CFG, + AN7583_I2C1_SCL_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio42", GPIO42_FLASH_MODE_CFG, + AN7583_I2C1_SDA_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio43", GPIO43_FLASH_MODE_CFG, + AN7583_SPI_CLK_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio44", GPIO44_FLASH_MODE_CFG, + AN7583_SPI_CS_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio45", GPIO45_FLASH_MODE_CFG, + AN7583_SPI_MOSI_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio46", GPIO46_FLASH_MODE_CFG, + AN7583_SPI_MISO_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio47", GPIO47_FLASH_MODE_CFG, + AN7583_UART_TXD_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio48", GPIO48_FLASH_MODE_CFG, + AN7583_UART_RXD_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio49", GPIO49_FLASH_MODE_CFG, + GPIO_PCIE_RESET0_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio50", GPIO50_FLASH_MODE_CFG, + GPIO_PCIE_RESET1_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio51", GPIO51_FLASH_MODE_CFG, + AN7583_MDC_0_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO("gpio52", AN7583_MDIO_0_GPIO_MODE_MASK), +}; + static const struct airoha_pinctrl_func_group an7583_pcie_reset_func_group[] = { { .name = "pcie_reset0", @@ -1443,7 +1547,7 @@ static const struct airoha_pinctrl_func_group an7583_pcie_reset_func_group[] = { AIROHA_FUNC_MUX, REG_GPIO_PON_MODE, GPIO_PCIE_RESET0_MASK, - GPIO_PCIE_RESET0_MASK + 0 }, .regmap_size = 1, }, { @@ -1452,7 +1556,7 @@ static const struct airoha_pinctrl_func_group an7583_pcie_reset_func_group[] = { AIROHA_FUNC_MUX, REG_GPIO_PON_MODE, GPIO_PCIE_RESET1_MASK, - GPIO_PCIE_RESET1_MASK + 0 }, .regmap_size = 1, }, @@ -1483,6 +1587,24 @@ static const struct airoha_pinctrl_func_group an7583_pcie_reset_func_group[] = { .regmap_size = 1, \ } \ +#define AIROHA_PINCTRL_PWM_EXT_SEC(gpio, mux_val, smux_val) \ + { \ + .name = (gpio), \ + .regmap[0] = { \ + AIROHA_FUNC_PWM_EXT_MUX, \ + REG_GPIO_FLASH_MODE_CFG_EXT, \ + (mux_val), \ + (mux_val) \ + }, \ + .regmap[1] = { \ + AIROHA_FUNC_MUX, \ + REG_GPIO_PON_MODE, \ + (smux_val), \ + (smux_val) \ + }, \ + .regmap_size = 2, \ + } + static const struct airoha_pinctrl_func_group pwm_func_group[] = { AIROHA_PINCTRL_PWM("gpio0", GPIO0_FLASH_MODE_CFG), AIROHA_PINCTRL_PWM("gpio1", GPIO1_FLASH_MODE_CFG), @@ -1527,6 +1649,12 @@ static const struct airoha_pinctrl_func_group pwm_func_group[] = { AIROHA_PINCTRL_PWM_EXT("gpio44", GPIO44_FLASH_MODE_CFG), AIROHA_PINCTRL_PWM_EXT("gpio45", GPIO45_FLASH_MODE_CFG), AIROHA_PINCTRL_PWM_EXT("gpio46", GPIO46_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio47", GPIO47_FLASH_MODE_CFG, + GPIO_PCIE_RESET0_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio48", GPIO48_FLASH_MODE_CFG, + GPIO_PCIE_RESET1_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio49", GPIO49_FLASH_MODE_CFG, + GPIO_PCIE_RESET2_MASK), }; static const struct airoha_pinctrl_func_group an7583_pwm_func_group[] = { @@ -1565,16 +1693,32 @@ static const struct airoha_pinctrl_func_group an7583_pwm_func_group[] = { AIROHA_PINCTRL_PWM_EXT("gpio36", GPIO36_FLASH_MODE_CFG), AIROHA_PINCTRL_PWM_EXT("gpio37", GPIO37_FLASH_MODE_CFG), AIROHA_PINCTRL_PWM_EXT("gpio38", GPIO38_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio39", GPIO39_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio40", GPIO40_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio41", GPIO41_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio42", GPIO42_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio43", GPIO43_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio44", GPIO44_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio45", GPIO45_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio46", GPIO46_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio47", GPIO47_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio48", GPIO48_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio39", GPIO39_FLASH_MODE_CFG, + AN7583_I2C0_SCL_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio40", GPIO40_FLASH_MODE_CFG, + AN7583_I2C0_SDA_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio41", GPIO41_FLASH_MODE_CFG, + AN7583_I2C1_SCL_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio42", GPIO42_FLASH_MODE_CFG, + AN7583_I2C1_SDA_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio43", GPIO43_FLASH_MODE_CFG, + AN7583_SPI_CLK_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio44", GPIO44_FLASH_MODE_CFG, + AN7583_SPI_CS_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio45", GPIO45_FLASH_MODE_CFG, + AN7583_SPI_MOSI_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio46", GPIO46_FLASH_MODE_CFG, + AN7583_SPI_MISO_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio47", GPIO47_FLASH_MODE_CFG, + AN7583_UART_TXD_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio48", GPIO48_FLASH_MODE_CFG, + AN7583_UART_RXD_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio49", GPIO49_FLASH_MODE_CFG, + GPIO_PCIE_RESET0_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio50", GPIO50_FLASH_MODE_CFG, + GPIO_PCIE_RESET1_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio51", GPIO51_FLASH_MODE_CFG, + AN7583_MDC_0_GPIO_MODE_MASK), }; #define AIROHA_PINCTRL_PHY_LED0(gpio, mux_val, map_mask, map_val) \ @@ -1803,6 +1947,7 @@ static const struct airoha_pinctrl_func en7581_pinctrl_funcs[] = { PINCTRL_FUNC_DESC("i2s", i2s), PINCTRL_FUNC_DESC("emmc", emmc), PINCTRL_FUNC_DESC("pnand", pnand), + PINCTRL_FUNC_DESC("gpio", gpio), PINCTRL_FUNC_DESC("pcie_reset", pcie_reset), PINCTRL_FUNC_DESC("pwm", pwm), PINCTRL_FUNC_DESC("phy1_led0", phy1_led0), @@ -1828,6 +1973,7 @@ static const struct airoha_pinctrl_func an7583_pinctrl_funcs[] = { PINCTRL_FUNC_DESC("pcm_spi", an7583_pcm_spi), PINCTRL_FUNC_DESC("emmc", emmc), PINCTRL_FUNC_DESC("pnand", pnand), + PINCTRL_FUNC_DESC("gpio", an7583_gpio), PINCTRL_FUNC_DESC("pcie_reset", an7583_pcie_reset), PINCTRL_FUNC_DESC("pwm", an7583_pwm), PINCTRL_FUNC_DESC("phy1_led0", an7583_phy1_led0), @@ -1843,8 +1989,8 @@ static const struct airoha_pinctrl_func an7583_pinctrl_funcs[] = { static const struct airoha_pinctrl_conf en7581_pinctrl_pullup_conf[] = { PINCTRL_CONF_DESC(0, REG_I2C_SDA_PU, UART1_TXD_PU_MASK), PINCTRL_CONF_DESC(1, REG_I2C_SDA_PU, UART1_RXD_PU_MASK), - PINCTRL_CONF_DESC(2, REG_I2C_SDA_PU, I2C_SDA_PU_MASK), - PINCTRL_CONF_DESC(3, REG_I2C_SDA_PU, I2C_SCL_PU_MASK), + PINCTRL_CONF_DESC(2, REG_I2C_SDA_PU, I2C_SCL_PU_MASK), + PINCTRL_CONF_DESC(3, REG_I2C_SDA_PU, I2C_SDA_PU_MASK), PINCTRL_CONF_DESC(4, REG_I2C_SDA_PU, SPI_CS0_PU_MASK), PINCTRL_CONF_DESC(5, REG_I2C_SDA_PU, SPI_CLK_PU_MASK), PINCTRL_CONF_DESC(6, REG_I2C_SDA_PU, SPI_MOSI_PU_MASK), @@ -1896,9 +2042,9 @@ static const struct airoha_pinctrl_conf en7581_pinctrl_pullup_conf[] = { PINCTRL_CONF_DESC(57, REG_GPIO_H_PU, BIT(12)), PINCTRL_CONF_DESC(58, REG_GPIO_H_PU, BIT(13)), PINCTRL_CONF_DESC(59, REG_GPIO_H_PU, BIT(14)), - PINCTRL_CONF_DESC(61, REG_I2C_SDA_PU, PCIE0_RESET_PU_MASK), - PINCTRL_CONF_DESC(62, REG_I2C_SDA_PU, PCIE1_RESET_PU_MASK), - PINCTRL_CONF_DESC(63, REG_I2C_SDA_PU, PCIE2_RESET_PU_MASK), + PINCTRL_CONF_DESC(60, REG_I2C_SDA_PU, PCIE0_RESET_PU_MASK), + PINCTRL_CONF_DESC(61, REG_I2C_SDA_PU, PCIE1_RESET_PU_MASK), + PINCTRL_CONF_DESC(62, REG_I2C_SDA_PU, PCIE2_RESET_PU_MASK), }; static const struct airoha_pinctrl_conf an7583_pinctrl_pullup_conf[] = { @@ -1960,8 +2106,8 @@ static const struct airoha_pinctrl_conf an7583_pinctrl_pullup_conf[] = { static const struct airoha_pinctrl_conf en7581_pinctrl_pulldown_conf[] = { PINCTRL_CONF_DESC(0, REG_I2C_SDA_PD, UART1_TXD_PD_MASK), PINCTRL_CONF_DESC(1, REG_I2C_SDA_PD, UART1_RXD_PD_MASK), - PINCTRL_CONF_DESC(2, REG_I2C_SDA_PD, I2C_SDA_PD_MASK), - PINCTRL_CONF_DESC(3, REG_I2C_SDA_PD, I2C_SCL_PD_MASK), + PINCTRL_CONF_DESC(2, REG_I2C_SDA_PD, I2C_SCL_PD_MASK), + PINCTRL_CONF_DESC(3, REG_I2C_SDA_PD, I2C_SDA_PD_MASK), PINCTRL_CONF_DESC(4, REG_I2C_SDA_PD, SPI_CS0_PD_MASK), PINCTRL_CONF_DESC(5, REG_I2C_SDA_PD, SPI_CLK_PD_MASK), PINCTRL_CONF_DESC(6, REG_I2C_SDA_PD, SPI_MOSI_PD_MASK), @@ -2013,9 +2159,9 @@ static const struct airoha_pinctrl_conf en7581_pinctrl_pulldown_conf[] = { PINCTRL_CONF_DESC(57, REG_GPIO_H_PD, BIT(12)), PINCTRL_CONF_DESC(58, REG_GPIO_H_PD, BIT(13)), PINCTRL_CONF_DESC(59, REG_GPIO_H_PD, BIT(14)), - PINCTRL_CONF_DESC(61, REG_I2C_SDA_PD, PCIE0_RESET_PD_MASK), - PINCTRL_CONF_DESC(62, REG_I2C_SDA_PD, PCIE1_RESET_PD_MASK), - PINCTRL_CONF_DESC(63, REG_I2C_SDA_PD, PCIE2_RESET_PD_MASK), + PINCTRL_CONF_DESC(60, REG_I2C_SDA_PD, PCIE0_RESET_PD_MASK), + PINCTRL_CONF_DESC(61, REG_I2C_SDA_PD, PCIE1_RESET_PD_MASK), + PINCTRL_CONF_DESC(62, REG_I2C_SDA_PD, PCIE2_RESET_PD_MASK), }; static const struct airoha_pinctrl_conf an7583_pinctrl_pulldown_conf[] = { @@ -2077,8 +2223,8 @@ static const struct airoha_pinctrl_conf an7583_pinctrl_pulldown_conf[] = { static const struct airoha_pinctrl_conf en7581_pinctrl_drive_e2_conf[] = { PINCTRL_CONF_DESC(0, REG_I2C_SDA_E2, UART1_TXD_E2_MASK), PINCTRL_CONF_DESC(1, REG_I2C_SDA_E2, UART1_RXD_E2_MASK), - PINCTRL_CONF_DESC(2, REG_I2C_SDA_E2, I2C_SDA_E2_MASK), - PINCTRL_CONF_DESC(3, REG_I2C_SDA_E2, I2C_SCL_E2_MASK), + PINCTRL_CONF_DESC(2, REG_I2C_SDA_E2, I2C_SCL_E2_MASK), + PINCTRL_CONF_DESC(3, REG_I2C_SDA_E2, I2C_SDA_E2_MASK), PINCTRL_CONF_DESC(4, REG_I2C_SDA_E2, SPI_CS0_E2_MASK), PINCTRL_CONF_DESC(5, REG_I2C_SDA_E2, SPI_CLK_E2_MASK), PINCTRL_CONF_DESC(6, REG_I2C_SDA_E2, SPI_MOSI_E2_MASK), @@ -2130,9 +2276,9 @@ static const struct airoha_pinctrl_conf en7581_pinctrl_drive_e2_conf[] = { PINCTRL_CONF_DESC(57, REG_GPIO_H_E2, BIT(12)), PINCTRL_CONF_DESC(58, REG_GPIO_H_E2, BIT(13)), PINCTRL_CONF_DESC(59, REG_GPIO_H_E2, BIT(14)), - PINCTRL_CONF_DESC(61, REG_I2C_SDA_E2, PCIE0_RESET_E2_MASK), - PINCTRL_CONF_DESC(62, REG_I2C_SDA_E2, PCIE1_RESET_E2_MASK), - PINCTRL_CONF_DESC(63, REG_I2C_SDA_E2, PCIE2_RESET_E2_MASK), + PINCTRL_CONF_DESC(60, REG_I2C_SDA_E2, PCIE0_RESET_E2_MASK), + PINCTRL_CONF_DESC(61, REG_I2C_SDA_E2, PCIE1_RESET_E2_MASK), + PINCTRL_CONF_DESC(62, REG_I2C_SDA_E2, PCIE2_RESET_E2_MASK), }; static const struct airoha_pinctrl_conf an7583_pinctrl_drive_e2_conf[] = { @@ -2194,8 +2340,8 @@ static const struct airoha_pinctrl_conf an7583_pinctrl_drive_e2_conf[] = { static const struct airoha_pinctrl_conf en7581_pinctrl_drive_e4_conf[] = { PINCTRL_CONF_DESC(0, REG_I2C_SDA_E4, UART1_TXD_E4_MASK), PINCTRL_CONF_DESC(1, REG_I2C_SDA_E4, UART1_RXD_E4_MASK), - PINCTRL_CONF_DESC(2, REG_I2C_SDA_E4, I2C_SDA_E4_MASK), - PINCTRL_CONF_DESC(3, REG_I2C_SDA_E4, I2C_SCL_E4_MASK), + PINCTRL_CONF_DESC(2, REG_I2C_SDA_E4, I2C_SCL_E4_MASK), + PINCTRL_CONF_DESC(3, REG_I2C_SDA_E4, I2C_SDA_E4_MASK), PINCTRL_CONF_DESC(4, REG_I2C_SDA_E4, SPI_CS0_E4_MASK), PINCTRL_CONF_DESC(5, REG_I2C_SDA_E4, SPI_CLK_E4_MASK), PINCTRL_CONF_DESC(6, REG_I2C_SDA_E4, SPI_MOSI_E4_MASK), @@ -2247,9 +2393,9 @@ static const struct airoha_pinctrl_conf en7581_pinctrl_drive_e4_conf[] = { PINCTRL_CONF_DESC(57, REG_GPIO_H_E4, BIT(12)), PINCTRL_CONF_DESC(58, REG_GPIO_H_E4, BIT(13)), PINCTRL_CONF_DESC(59, REG_GPIO_H_E4, BIT(14)), - PINCTRL_CONF_DESC(61, REG_I2C_SDA_E4, PCIE0_RESET_E4_MASK), - PINCTRL_CONF_DESC(62, REG_I2C_SDA_E4, PCIE1_RESET_E4_MASK), - PINCTRL_CONF_DESC(63, REG_I2C_SDA_E4, PCIE2_RESET_E4_MASK), + PINCTRL_CONF_DESC(60, REG_I2C_SDA_E4, PCIE0_RESET_E4_MASK), + PINCTRL_CONF_DESC(61, REG_I2C_SDA_E4, PCIE1_RESET_E4_MASK), + PINCTRL_CONF_DESC(62, REG_I2C_SDA_E4, PCIE2_RESET_E4_MASK), }; static const struct airoha_pinctrl_conf an7583_pinctrl_drive_e4_conf[] = { @@ -2309,9 +2455,9 @@ static const struct airoha_pinctrl_conf an7583_pinctrl_drive_e4_conf[] = { }; static const struct airoha_pinctrl_conf en7581_pinctrl_pcie_rst_od_conf[] = { - PINCTRL_CONF_DESC(61, REG_PCIE_RESET_OD, PCIE0_RESET_OD_MASK), - PINCTRL_CONF_DESC(62, REG_PCIE_RESET_OD, PCIE1_RESET_OD_MASK), - PINCTRL_CONF_DESC(63, REG_PCIE_RESET_OD, PCIE2_RESET_OD_MASK), + PINCTRL_CONF_DESC(60, REG_PCIE_RESET_OD, PCIE0_RESET_OD_MASK), + PINCTRL_CONF_DESC(61, REG_PCIE_RESET_OD, PCIE1_RESET_OD_MASK), + PINCTRL_CONF_DESC(62, REG_PCIE_RESET_OD, PCIE2_RESET_OD_MASK), }; static const struct airoha_pinctrl_conf an7583_pinctrl_pcie_rst_od_conf[] = { @@ -2358,6 +2504,22 @@ static int airoha_gpio_get(struct gpio_chip *chip, unsigned int gpio) return err ? err : !!(val & BIT(pin)); } +static int airoha_gpio_get_direction(struct gpio_chip *chip, unsigned int gpio) +{ + struct airoha_pinctrl *pinctrl = gpiochip_get_data(chip); + u32 val, mask; + u8 index; + int err; + + index = gpio / AIROHA_REG_GPIOCTRL_NUM_PIN; + err = regmap_read(pinctrl->regmap, pinctrl->gpiochip.dir[index], &val); + if (err) + return err; + + mask = BIT(2 * (gpio % AIROHA_REG_GPIOCTRL_NUM_PIN)); + return val & mask ? GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN; +} + static int airoha_gpio_direction_output(struct gpio_chip *chip, unsigned int gpio, int value) { @@ -2373,18 +2535,18 @@ static int airoha_gpio_direction_output(struct gpio_chip *chip, /* irq callbacks */ static void airoha_irq_unmask(struct irq_data *data) { + struct gpio_chip *gc = irq_data_get_irq_chip_data(data); + struct airoha_pinctrl *pinctrl = gpiochip_get_data(gc); + struct airoha_pinctrl_gpiochip *gpiochip = &pinctrl->gpiochip; u8 offset = data->hwirq % AIROHA_REG_GPIOCTRL_NUM_PIN; u8 index = data->hwirq / AIROHA_REG_GPIOCTRL_NUM_PIN; u32 mask = GENMASK(2 * offset + 1, 2 * offset); - struct airoha_pinctrl_gpiochip *gpiochip; - struct airoha_pinctrl *pinctrl; u32 val = BIT(2 * offset); - gpiochip = irq_data_get_irq_chip_data(data); if (WARN_ON_ONCE(data->hwirq >= ARRAY_SIZE(gpiochip->irq_type))) return; - pinctrl = container_of(gpiochip, struct airoha_pinctrl, gpiochip); + gpiochip_enable_irq(gc, irqd_to_hwirq(data)); switch (gpiochip->irq_type[data->hwirq]) { case IRQ_TYPE_LEVEL_LOW: val = val << 1; @@ -2410,34 +2572,64 @@ static void airoha_irq_unmask(struct irq_data *data) static void airoha_irq_mask(struct irq_data *data) { + struct gpio_chip *gc = irq_data_get_irq_chip_data(data); + struct airoha_pinctrl *pinctrl = gpiochip_get_data(gc); + struct airoha_pinctrl_gpiochip *gpiochip = &pinctrl->gpiochip; u8 offset = data->hwirq % AIROHA_REG_GPIOCTRL_NUM_PIN; u8 index = data->hwirq / AIROHA_REG_GPIOCTRL_NUM_PIN; u32 mask = GENMASK(2 * offset + 1, 2 * offset); - struct airoha_pinctrl_gpiochip *gpiochip; - struct airoha_pinctrl *pinctrl; - gpiochip = irq_data_get_irq_chip_data(data); - pinctrl = container_of(gpiochip, struct airoha_pinctrl, gpiochip); + if (data->hwirq >= ARRAY_SIZE(gpiochip->irq_type)) + return; regmap_clear_bits(pinctrl->regmap, gpiochip->level[index], mask); regmap_clear_bits(pinctrl->regmap, gpiochip->edge[index], mask); + gpiochip_disable_irq(gc, irqd_to_hwirq(data)); +} + +static void airoha_irq_ack(struct irq_data *data) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(data); + struct airoha_pinctrl *pinctrl = gpiochip_get_data(gc); + struct airoha_pinctrl_gpiochip *gpiochip = &pinctrl->gpiochip; + u8 offset = data->hwirq % AIROHA_PIN_BANK_SIZE; + u8 index = data->hwirq / AIROHA_PIN_BANK_SIZE; + + if (data->hwirq >= ARRAY_SIZE(gpiochip->irq_type)) + return; + + regmap_write(pinctrl->regmap, gpiochip->status[index], BIT(offset)); } static int airoha_irq_type(struct irq_data *data, unsigned int type) { - struct airoha_pinctrl_gpiochip *gpiochip; + struct gpio_chip *gc = irq_data_get_irq_chip_data(data); + struct airoha_pinctrl *pinctrl = gpiochip_get_data(gc); + struct airoha_pinctrl_gpiochip *gpiochip = &pinctrl->gpiochip; - gpiochip = irq_data_get_irq_chip_data(data); if (data->hwirq >= ARRAY_SIZE(gpiochip->irq_type)) return -EINVAL; + if (type == IRQ_TYPE_NONE) { + gpiochip->irq_type[data->hwirq] = IRQ_TYPE_NONE; + irq_set_handler_locked(data, handle_bad_irq); + + return 0; + } + if (type == IRQ_TYPE_PROBE) { if (gpiochip->irq_type[data->hwirq]) return 0; type = IRQ_TYPE_EDGE_RISING | IRQ_TYPE_EDGE_FALLING; } + gpiochip->irq_type[data->hwirq] = type & IRQ_TYPE_SENSE_MASK; + if (type & IRQ_TYPE_EDGE_BOTH) + irq_set_handler_locked(data, handle_edge_irq); + else + irq_set_handler_locked(data, handle_level_irq); + return 0; } @@ -2462,8 +2654,7 @@ static irqreturn_t airoha_irq_handler(int irq, void *data) for_each_set_bit(irq, &status, AIROHA_PIN_BANK_SIZE) { u32 offset = irq + i * AIROHA_PIN_BANK_SIZE; - generic_handle_irq(irq_find_mapping(girq->domain, - offset)); + generic_handle_domain_irq(girq->domain, offset); regmap_write(pinctrl->regmap, pinctrl->gpiochip.status[i], BIT(irq)); } @@ -2477,9 +2668,10 @@ static const struct irq_chip airoha_gpio_irq_chip = { .name = "airoha-gpio-irq", .irq_unmask = airoha_irq_unmask, .irq_mask = airoha_irq_mask, - .irq_mask_ack = airoha_irq_mask, + .irq_ack = airoha_irq_ack, .irq_set_type = airoha_irq_type, .flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_IMMUTABLE, + GPIOCHIP_IRQ_RESOURCE_HELPERS, }; static int airoha_pinctrl_add_gpiochip(struct airoha_pinctrl *pinctrl, @@ -2491,26 +2683,20 @@ static int airoha_pinctrl_add_gpiochip(struct airoha_pinctrl *pinctrl, struct device *dev = &pdev->dev; int irq, err; - chip->data = gpio_data_regs; - chip->dir = gpio_dir_regs; - chip->out = gpio_out_regs; - chip->status = irq_status_regs; - chip->level = irq_level_regs; - chip->edge = irq_edge_regs; - gc->parent = dev; gc->label = dev_name(dev); gc->request = gpiochip_generic_request; gc->free = gpiochip_generic_free; gc->direction_input = pinctrl_gpio_direction_input; gc->direction_output = airoha_gpio_direction_output; + gc->get_direction = airoha_gpio_get_direction; gc->set = airoha_gpio_set; gc->get = airoha_gpio_get; gc->base = -1; gc->ngpio = AIROHA_NUM_PINS; girq->default_type = IRQ_TYPE_NONE; - girq->handler = handle_simple_irq; + girq->handler = handle_bad_irq; gpio_irq_chip_set_chip(girq, &airoha_gpio_irq_chip); irq = platform_get_irq(pdev, 0); @@ -2713,21 +2899,18 @@ static int airoha_pinctrl_set_conf(struct airoha_pinctrl *pinctrl, static int airoha_pinconf_get_direction(struct pinctrl_dev *pctrl_dev, u32 p) { struct airoha_pinctrl *pinctrl = pinctrl_dev_get_drvdata(pctrl_dev); - u32 val, mask; - int err, pin; - u8 index; + int ret, pin; pin = airoha_convert_pin_to_reg_offset(pctrl_dev, NULL, p); if (pin < 0) return pin; - index = pin / AIROHA_REG_GPIOCTRL_NUM_PIN; - err = regmap_read(pinctrl->regmap, pinctrl->gpiochip.dir[index], &val); - if (err) - return err; + ret = airoha_gpio_get_direction(&pinctrl->gpiochip.chip, pin); + if (ret < 0) + return ret; - mask = BIT(2 * (pin % AIROHA_REG_GPIOCTRL_NUM_PIN)); - return val & mask ? PIN_CONFIG_OUTPUT_ENABLE : PIN_CONFIG_INPUT_ENABLE; + return ret == GPIO_LINE_DIRECTION_OUT ? + PIN_CONFIG_OUTPUT_ENABLE : PIN_CONFIG_INPUT_ENABLE; } static int airoha_pinconf_get(struct pinctrl_dev *pctrl_dev, @@ -2973,6 +3156,17 @@ static int airoha_pinctrl_probe(struct platform_device *pdev) pinctrl->desc.pins = data->pins; pinctrl->desc.npins = data->num_pins; + /* + * some pinctrl operations (ex: get_direction) might use gpio registers + * before gpio chip abstraction will be completely initialized. + */ + pinctrl->gpiochip.data = gpio_data_regs; + pinctrl->gpiochip.dir = gpio_dir_regs; + pinctrl->gpiochip.out = gpio_out_regs; + pinctrl->gpiochip.status = irq_status_regs; + pinctrl->gpiochip.level = irq_level_regs; + pinctrl->gpiochip.edge = irq_edge_regs; + err = devm_pinctrl_register_and_init(dev, &pinctrl->desc, pinctrl, &pinctrl->ctrl); if (err) diff --git a/drivers/pinctrl/bcm/Kconfig b/drivers/pinctrl/bcm/Kconfig index 206f3f1249cf50..19d22e7fd11ed3 100644 --- a/drivers/pinctrl/bcm/Kconfig +++ b/drivers/pinctrl/bcm/Kconfig @@ -121,6 +121,7 @@ source "drivers/pinctrl/bcm/Kconfig.stb" config PINCTRL_IPROC_GPIO bool "Broadcom iProc GPIO (with PINCONF) driver" depends on ARCH_BCM_IPROC || COMPILE_TEST + select GPIOLIB select GPIOLIB_IRQCHIP select PINCONF select GENERIC_PINCONF @@ -186,6 +187,7 @@ config PINCTRL_NS config PINCTRL_NSP_GPIO bool "Broadcom NSP GPIO (with PINCONF) driver" depends on ARCH_BCM_NSP || COMPILE_TEST + select GPIOLIB select GPIOLIB_IRQCHIP select PINCONF select GENERIC_PINCONF diff --git a/drivers/pinctrl/bcm/pinctrl-bcm2835.c b/drivers/pinctrl/bcm/pinctrl-bcm2835.c index e7b35019a5a7d8..725e880ae086f8 100644 --- a/drivers/pinctrl/bcm/pinctrl-bcm2835.c +++ b/drivers/pinctrl/bcm/pinctrl-bcm2835.c @@ -1350,7 +1350,6 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev) pc->pctl_desc = *pdata->pctl_desc; pc->pctl_dev = devm_pinctrl_register(dev, &pc->pctl_desc, pc); if (IS_ERR(pc->pctl_dev)) { - gpiochip_remove(&pc->gpio_chip); return PTR_ERR(pc->pctl_dev); } diff --git a/drivers/pinctrl/freescale/pinctrl-imx1-core.c b/drivers/pinctrl/freescale/pinctrl-imx1-core.c index b7bd4ef9c0db5f..4a6bdaefa42f20 100644 --- a/drivers/pinctrl/freescale/pinctrl-imx1-core.c +++ b/drivers/pinctrl/freescale/pinctrl-imx1-core.c @@ -547,14 +547,11 @@ static int imx1_pinctrl_parse_functions(struct device_node *np, */ static bool imx1_pinctrl_dt_is_flat_functions(struct device_node *np) { - struct device_node *function_np; - struct device_node *pinctrl_np; - - for_each_child_of_node(np, function_np) { + for_each_child_of_node_scoped(np, function_np) { if (of_property_present(function_np, "fsl,pins")) return true; - for_each_child_of_node(function_np, pinctrl_np) { + for_each_child_of_node_scoped(function_np, pinctrl_np) { if (of_property_present(pinctrl_np, "fsl,pins")) return false; } diff --git a/drivers/pinctrl/mediatek/mtk-eint.c b/drivers/pinctrl/mediatek/mtk-eint.c index 47ac92ea98c2c5..8b022545a3e9f7 100644 --- a/drivers/pinctrl/mediatek/mtk-eint.c +++ b/drivers/pinctrl/mediatek/mtk-eint.c @@ -12,8 +12,10 @@ */ #include +#include #include #include +#include #include #include #include @@ -509,6 +511,27 @@ int mtk_eint_find_irq(struct mtk_eint *eint, unsigned long eint_n) } EXPORT_SYMBOL_GPL(mtk_eint_find_irq); +static void mtk_eint_teardown(void *data) +{ + struct mtk_eint *eint = data; + unsigned int i, virq; + + /* Detach the demux handler so it can no longer reference freed data. */ + irq_set_chained_handler_and_data(eint->irq, NULL, NULL); + + /* Wait for any in-flight handler to finish before tearing down. */ + synchronize_irq(eint->irq); + + /* Dispose of all child mappings before the domain is removed. */ + for (i = 0; i < eint->hw->ap_num; i++) { + virq = irq_find_mapping(eint->domain, i); + if (virq) + irq_dispose_mapping(virq); + } + + irq_domain_remove(eint->domain); +} + int mtk_eint_do_init(struct mtk_eint *eint, struct mtk_eint_pin *eint_pin) { unsigned int size, i, port, virq, inst = 0; @@ -601,7 +624,7 @@ int mtk_eint_do_init(struct mtk_eint *eint, struct mtk_eint_pin *eint_pin) irq_set_chained_handler_and_data(eint->irq, mtk_eint_irq_handler, eint); - return 0; + return devm_add_action_or_reset(eint->dev, mtk_eint_teardown, eint); err_eint: for (i = 0; i < eint->nbase; i++) { diff --git a/drivers/pinctrl/mediatek/pinctrl-moore.c b/drivers/pinctrl/mediatek/pinctrl-moore.c index 17e30f83dc1972..38f15dbe9a283b 100644 --- a/drivers/pinctrl/mediatek/pinctrl-moore.c +++ b/drivers/pinctrl/mediatek/pinctrl-moore.c @@ -594,7 +594,7 @@ static int mtk_build_gpiochip(struct mtk_pinctrl *hw) chip->base = -1; chip->ngpio = hw->soc->npins; - ret = gpiochip_add_data(chip, hw); + ret = devm_gpiochip_add_data(hw->dev, chip, hw); if (ret < 0) return ret; @@ -608,10 +608,8 @@ static int mtk_build_gpiochip(struct mtk_pinctrl *hw) if (!of_property_present(hw->dev->of_node, "gpio-ranges")) { ret = gpiochip_add_pin_range(chip, dev_name(hw->dev), 0, 0, chip->ngpio); - if (ret < 0) { - gpiochip_remove(chip); + if (ret < 0) return ret; - } } return 0; diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c index dd2c8aa0393857..791eddd7a2c63f 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c +++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c @@ -1130,30 +1130,24 @@ int mtk_pctrl_init(struct platform_device *pdev, pctl->chip->parent = &pdev->dev; pctl->chip->base = -1; - ret = gpiochip_add_data(pctl->chip, pctl); + ret = devm_gpiochip_add_data(&pdev->dev, pctl->chip, pctl); if (ret) return -EINVAL; /* Register the GPIO to pin mappings. */ ret = gpiochip_add_pin_range(pctl->chip, dev_name(&pdev->dev), 0, 0, pctl->devdata->npins); - if (ret) { - ret = -EINVAL; - goto chip_error; - } + if (ret) + return -EINVAL; /* Only initialize EINT if we have EINT pins */ if (data->eint_hw.ap_num > 0) { ret = mtk_eint_init(pctl, pdev); if (ret) - goto chip_error; + return ret; } return 0; - -chip_error: - gpiochip_remove(pctl->chip); - return ret; } int mtk_pctrl_common_probe(struct platform_device *pdev) diff --git a/drivers/pinctrl/mediatek/pinctrl-paris.c b/drivers/pinctrl/mediatek/pinctrl-paris.c index 23f04b24fd65ef..09098b68f72575 100644 --- a/drivers/pinctrl/mediatek/pinctrl-paris.c +++ b/drivers/pinctrl/mediatek/pinctrl-paris.c @@ -957,7 +957,7 @@ static int mtk_build_gpiochip(struct mtk_pinctrl *hw) chip->base = -1; chip->ngpio = hw->soc->npins; - ret = gpiochip_add_data(chip, hw); + ret = devm_gpiochip_add_data(hw->dev, chip, hw); if (ret < 0) return ret; diff --git a/drivers/pinctrl/pinctrl-eic7700.c b/drivers/pinctrl/pinctrl-eic7700.c index d553ec20c61914..09a3b097383cbb 100644 --- a/drivers/pinctrl/pinctrl-eic7700.c +++ b/drivers/pinctrl/pinctrl-eic7700.c @@ -422,6 +422,9 @@ static int eic7700_pin_config_set(struct pinctrl_dev *pctldev, unsigned int pin, else value &= ~EIC7700_ST; break; + /* All pins are persistent over suspend */ + case PIN_CONFIG_PERSIST_STATE: + return 0; default: return -EOPNOTSUPP; } diff --git a/drivers/pinctrl/pinctrl-generic.c b/drivers/pinctrl/pinctrl-generic.c index 9759b0186bcc23..fd6bdb74028aa5 100644 --- a/drivers/pinctrl/pinctrl-generic.c +++ b/drivers/pinctrl/pinctrl-generic.c @@ -42,33 +42,44 @@ int pinctrl_generic_to_map(struct pinctrl_dev *pctldev, struct device_node *pare ret = pinctrl_utils_add_map_mux(pctldev, maps, num_reserved_maps, num_maps, group_name, parent->name); if (ret < 0) - return ret; + goto err_free_map; ret = pinctrl_generic_add_group(pctldev, group_name, pins, npins, data); - if (ret < 0) - return dev_err_probe(dev, ret, "failed to add group %s: %d\n", + if (ret < 0) { + dev_err_probe(dev, ret, "failed to add group %s: %d\n", group_name, ret); + goto err_free_map; + } ret = pinconf_generic_parse_dt_config(np, pctldev, &configs, &num_configs); - if (ret) - return dev_err_probe(dev, ret, "failed to parse pin config of group %s\n", + if (ret) { + dev_err_probe(dev, ret, "failed to parse pin config of group %s\n", group_name); + goto err_free_map; + } if (num_configs == 0) return 0; ret = pinctrl_utils_reserve_map(pctldev, maps, num_reserved_maps, num_maps, reserve); if (ret) - return ret; + goto err_free_map; ret = pinctrl_utils_add_map_configs(pctldev, maps, num_reserved_maps, num_maps, group_name, configs, num_configs, PIN_MAP_TYPE_CONFIGS_GROUP); kfree(configs); if (ret) - return ret; + goto err_free_map; return 0; + +err_free_map: + pinctrl_utils_free_map(pctldev, *maps, *num_maps); + *maps = NULL; + *num_maps = 0; + *num_reserved_maps = 0; + return ret; }; EXPORT_SYMBOL_GPL(pinctrl_generic_to_map); diff --git a/drivers/pinctrl/pinctrl-rockchip.c b/drivers/pinctrl/pinctrl-rockchip.c index 7e0fcd45fd2657..7f7ea3b3638d95 100644 --- a/drivers/pinctrl/pinctrl-rockchip.c +++ b/drivers/pinctrl/pinctrl-rockchip.c @@ -3212,7 +3212,7 @@ static int rockchip_get_drive_perpin(struct rockchip_pin_bank *bank, u8 bit; int drv_type = bank->drv[pin_num / 8].drv_type; - if (ctrl->type == RV1103B && pin_num >= 12) + if (ctrl->type == RV1103B && bank->bank_num == 2 && pin_num >= 12) drv_type = DRV_TYPE_IO_LEVEL_2_BIT; ret = ctrl->drv_calc_reg(bank, pin_num, ®map, ®, &bit); @@ -3267,6 +3267,25 @@ static int rockchip_get_drive_perpin(struct rockchip_pin_bank *bank, case DRV_TYPE_IO_1V8_ONLY: rmask_bits = RK3288_DRV_BITS_PER_PIN; break; + case DRV_TYPE_IO_LEVEL_2_BIT: + ret = regmap_read(regmap, reg, &data); + if (ret) + return ret; + data >>= bit; + + return data & 0x3; + case DRV_TYPE_IO_LEVEL_8_BIT: + ret = regmap_read(regmap, reg, &data); + if (ret) + return ret; + data >>= bit; + data &= (1 << 8) - 1; + + ret = hweight8(data); + if (ret > 0) + return ret - 1; + else + return -EINVAL; default: dev_err(dev, "unsupported pinctrl drive type: %d\n", drv_type); return -EINVAL; @@ -3390,25 +3409,6 @@ static int rockchip_set_drive_perpin(struct rockchip_pin_bank *bank, case DRV_TYPE_IO_1V8_ONLY: rmask_bits = RK3288_DRV_BITS_PER_PIN; break; - case DRV_TYPE_IO_LEVEL_2_BIT: - ret = regmap_read(regmap, reg, &data); - if (ret) - return ret; - data >>= bit; - - return data & 0x3; - case DRV_TYPE_IO_LEVEL_8_BIT: - ret = regmap_read(regmap, reg, &data); - if (ret) - return ret; - data >>= bit; - data &= (1 << 8) - 1; - - ret = hweight8(data); - if (ret > 0) - return ret - 1; - else - return -EINVAL; default: dev_err(dev, "unsupported pinctrl drive type: %d\n", drv_type); return -EINVAL; @@ -4296,6 +4296,16 @@ static struct rockchip_pin_ctrl *rockchip_pinctrl_get_soc_data( pmu_offs = ctrl->pmu_mux_offset; drv_pmu_offs = ctrl->pmu_drv_offset; drv_grf_offs = ctrl->grf_drv_offset; + + /* + * This function mutates the static per-SoC data. Most of it is + * idempotent: recalculated iomux and drv offsets anchor at the + * values calculated by a previous run. The pin count is not, so + * reset it here; otherwise it accumulates when the probe runs + * again after a probe deferral, shifting every bank's pin_base. + */ + ctrl->nr_pins = 0; + bank = ctrl->pin_banks; for (i = 0; i < ctrl->nr_banks; ++i, ++bank) { int bank_pins = 0; diff --git a/drivers/pinctrl/qcom/pinctrl-shikra.c b/drivers/pinctrl/qcom/pinctrl-shikra.c index 0fc98369948cbe..55aec2f675e6b0 100644 --- a/drivers/pinctrl/qcom/pinctrl-shikra.c +++ b/drivers/pinctrl/qcom/pinctrl-shikra.c @@ -44,6 +44,7 @@ .intr_status_bit = 0, \ .intr_wakeup_enable_bit = 7, \ .intr_wakeup_present_bit = 6, \ + .intr_target_width = 4, \ .intr_target_bit = 8, \ .intr_target_kpss_val = 3, \ .intr_raw_status_bit = 4, \ diff --git a/drivers/pinctrl/spacemit/pinctrl-k1.c b/drivers/pinctrl/spacemit/pinctrl-k1.c index f0b5ebd9e223c7..c3a7538783b8a4 100644 --- a/drivers/pinctrl/spacemit/pinctrl-k1.c +++ b/drivers/pinctrl/spacemit/pinctrl-k1.c @@ -503,13 +503,14 @@ static int spacemit_pinconf_get(struct pinctrl_dev *pctldev, unsigned int pin, unsigned long *config) { struct spacemit_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev); + const struct spacemit_pin *spin = spacemit_get_pin(pctrl, pin); int param = pinconf_to_config_param(*config); u32 value, arg = 0; - if (!pin) + if (!spin) return -EINVAL; - value = readl(spacemit_pin_to_reg(pctrl, pin)); + value = readl(spacemit_pin_to_reg(pctrl, spin->pin)); switch (param) { case PIN_CONFIG_SLEW_RATE: @@ -689,6 +690,11 @@ static int spacemit_pinconf_group_set(struct pinctrl_dev *pctldev, if (ret) return ret; + for (i = 0; i < group->grp.npins; i++) { + if (!spacemit_get_pin(pctrl, group->grp.pins[i])) + return -EINVAL; + } + for (i = 0; i < group->grp.npins; i++) spacemit_pin_set_config(pctrl, group->grp.pins[i], value); diff --git a/drivers/pinctrl/tegra/pinctrl-tegra264.c b/drivers/pinctrl/tegra/pinctrl-tegra264.c index be64fba34dce46..f8afb0bdb68a8b 100644 --- a/drivers/pinctrl/tegra/pinctrl-tegra264.c +++ b/drivers/pinctrl/tegra/pinctrl-tegra264.c @@ -96,8 +96,8 @@ enum { TEGRA_PIN_UART4_RTS_N_PV4, TEGRA_PIN_UART4_CTS_N_PV5, TEGRA_PIN_DAP2_CLK_PV6, - TEGRA_PIN_DAP2_DIN_PV7, - TEGRA_PIN_DAP2_DOUT_PW0, + TEGRA_PIN_DAP2_DIN_PW0, + TEGRA_PIN_DAP2_DOUT_PV7, TEGRA_PIN_DAP2_FS_PW1, TEGRA_PIN_GEN1_I2C_SCL_PW2, TEGRA_PIN_GEN1_I2C_SDA_PW3, @@ -329,8 +329,8 @@ static const struct pinctrl_pin_desc tegra264_main_pins[] = { PINCTRL_PIN(TEGRA_PIN_UART4_RTS_N_PV4, "UART4_RTS_N_PV4"), PINCTRL_PIN(TEGRA_PIN_UART4_CTS_N_PV5, "UART4_CTS_N_PV5"), PINCTRL_PIN(TEGRA_PIN_DAP2_CLK_PV6, "DAP2_CLK_PV6"), - PINCTRL_PIN(TEGRA_PIN_DAP2_DIN_PV7, "DAP2_DIN_PV7"), - PINCTRL_PIN(TEGRA_PIN_DAP2_DOUT_PW0, "DAP2_DOUT_PW0"), + PINCTRL_PIN(TEGRA_PIN_DAP2_DIN_PW0, "DAP2_DIN_PW0"), + PINCTRL_PIN(TEGRA_PIN_DAP2_DOUT_PV7, "DAP2_DOUT_PV7"), PINCTRL_PIN(TEGRA_PIN_DAP2_FS_PW1, "DAP2_FS_PW1"), PINCTRL_PIN(TEGRA_PIN_GEN1_I2C_SCL_PW2, "GEN1_I2C_SCL_PW2"), PINCTRL_PIN(TEGRA_PIN_GEN1_I2C_SDA_PW3, "GEN1_I2C_SDA_PW3"), @@ -827,12 +827,12 @@ static const unsigned int dap2_clk_pv6_pins[] = { TEGRA_PIN_DAP2_CLK_PV6, }; -static const unsigned int dap2_din_pv7_pins[] = { - TEGRA_PIN_DAP2_DIN_PV7, +static const unsigned int dap2_din_pw0_pins[] = { + TEGRA_PIN_DAP2_DIN_PW0, }; -static const unsigned int dap2_dout_pw0_pins[] = { - TEGRA_PIN_DAP2_DOUT_PW0, +static const unsigned int dap2_dout_pv7_pins[] = { + TEGRA_PIN_DAP2_DOUT_PV7, }; static const unsigned int dap2_fs_pw1_pins[] = { @@ -1834,8 +1834,8 @@ static const char * const tegra264_functions[] = { #define drive_gen1_i2c_sda_pw3 DRV_PINGROUP_ENTRY_Y(0x601c, 12, 4, 20, 4, -1, -1, -1, -1, 0) #define drive_dap2_fs_pw1 DRV_PINGROUP_ENTRY_Y(0x6044, 12, 4, 20, 4, -1, -1, -1, -1, 0) #define drive_dap2_clk_pv6 DRV_PINGROUP_ENTRY_Y(0x604c, 12, 4, 20, 4, -1, -1, -1, -1, 0) -#define drive_dap2_din_pv7 DRV_PINGROUP_ENTRY_Y(0x6054, 12, 4, 20, 4, -1, -1, -1, -1, 0) -#define drive_dap2_dout_pw0 DRV_PINGROUP_ENTRY_Y(0x605c, 12, 4, 20, 4, -1, -1, -1, -1, 0) +#define drive_dap2_din_pw0 DRV_PINGROUP_ENTRY_Y(0x6054, 12, 4, 20, 4, -1, -1, -1, -1, 0) +#define drive_dap2_dout_pv7 DRV_PINGROUP_ENTRY_Y(0x605c, 12, 4, 20, 4, -1, -1, -1, -1, 0) #define drive_pwm10_pv1 DRV_PINGROUP_ENTRY_Y(0x6064, 12, 4, 20, 4, -1, -1, -1, -1, 0) #define drive_soc_gpio170_pu0 DRV_PINGROUP_ENTRY_Y(0x606c, 12, 4, 20, 4, -1, -1, -1, -1, 0) #define drive_soc_gpio171_pu1 DRV_PINGROUP_ENTRY_Y(0x6074, 12, 4, 20, 4, -1, -1, -1, -1, 0) @@ -2051,8 +2051,8 @@ static const struct tegra_pingroup tegra264_main_groups[] = { PINGROUP(gen1_i2c_sda_pw3, I2C1_DAT, RSVD1, RSVD2, RSVD3, 0x6018, 0, Y, 5, 7, 6, 8, -1, 10, 11), PINGROUP(dap2_fs_pw1, I2S2_LRCK, RSVD1, RSVD2, RSVD3, 0x6040, 0, Y, 5, 7, 6, 8, -1, 10, 11), PINGROUP(dap2_clk_pv6, I2S2_SCLK, RSVD1, RSVD2, RSVD3, 0x6048, 0, Y, 5, 7, 6, 8, -1, 10, 11), - PINGROUP(dap2_din_pv7, I2S2_SDATA_OUT, RSVD1, RSVD2, RSVD3, 0x6050, 0, Y, 5, 7, 6, 8, -1, 10, 11), - PINGROUP(dap2_dout_pw0, I2S2_SDATA_IN, RSVD1, RSVD2, RSVD3, 0x6058, 0, Y, 5, 7, 6, 8, -1, 10, 11), + PINGROUP(dap2_din_pw0, I2S2_SDATA_IN, RSVD1, RSVD2, RSVD3, 0x6050, 0, Y, 5, 7, 6, 8, -1, 10, 11), + PINGROUP(dap2_dout_pv7, I2S2_SDATA_OUT, RSVD1, RSVD2, RSVD3, 0x6058, 0, Y, 5, 7, 6, 8, -1, 10, 11), PINGROUP(pwm10_pv1, GP_PWM10, SDMMC1_CD, I2S7_LRCK, RSVD3, 0x6060, 0, Y, 5, 7, 6, 8, -1, 10, 11), PINGROUP(soc_gpio170_pu0, RSVD0, I2S7_SDATA_IN, CCLA_LA_TRIGGER_MUX, RSVD3, 0x6068, 0, Y, 5, 7, 6, 8, -1, 10, 11), PINGROUP(soc_gpio171_pu1, RSVD0, SPI4_SCK, RSVD2, RSVD3, 0x6070, 0, Y, 5, 7, 6, 8, -1, 10, 11), diff --git a/drivers/platform/arm64/qcom-hamoa-ec.c b/drivers/platform/arm64/qcom-hamoa-ec.c index 5ca7308c60774f..4d2ad042a7f846 100644 --- a/drivers/platform/arm64/qcom-hamoa-ec.c +++ b/drivers/platform/arm64/qcom-hamoa-ec.c @@ -92,8 +92,10 @@ static int qcom_ec_read(struct qcom_ec *ec, u8 cmd, u8 resp_len, u8 *resp) return ret; else if (ret == 0 || ret == 0xff) return -EOPNOTSUPP; + else if (ret != resp_len) + return -EIO; - if (resp[0] >= resp_len) + if (resp[0] != resp_len - 1) return -EINVAL; return 0; diff --git a/drivers/platform/chrome/cros_ec_debugfs.c b/drivers/platform/chrome/cros_ec_debugfs.c index 139cab6fcba174..c0cb50bd24413c 100644 --- a/drivers/platform/chrome/cros_ec_debugfs.c +++ b/drivers/platform/chrome/cros_ec_debugfs.c @@ -512,7 +512,7 @@ static int cros_ec_debugfs_probe(struct platform_device *pd) ret = blocking_notifier_chain_register(&ec->ec_dev->panic_notifier, &debug_info->notifier_panic); if (ret) - goto remove_debugfs; + goto cleanup_console_log; ec->debug_info = debug_info; @@ -520,6 +520,8 @@ static int cros_ec_debugfs_probe(struct platform_device *pd) return 0; +cleanup_console_log: + cros_ec_cleanup_console_log(debug_info); remove_debugfs: debugfs_remove_recursive(debug_info->dir); return ret; @@ -529,6 +531,8 @@ static void cros_ec_debugfs_remove(struct platform_device *pd) { struct cros_ec_dev *ec = dev_get_drvdata(pd->dev.parent); + blocking_notifier_chain_unregister(&ec->ec_dev->panic_notifier, + &ec->debug_info->notifier_panic); debugfs_remove_recursive(ec->debug_info->dir); cros_ec_cleanup_console_log(ec->debug_info); } diff --git a/drivers/platform/chrome/cros_ec_lightbar.c b/drivers/platform/chrome/cros_ec_lightbar.c index ac919c14c631e6..1a89e90957cdef 100644 --- a/drivers/platform/chrome/cros_ec_lightbar.c +++ b/drivers/platform/chrome/cros_ec_lightbar.c @@ -504,9 +504,14 @@ static ssize_t program_store(struct device *dev, struct device_attribute *attr, return -EINVAL; } } else { + /* + * Bound the payload strictly by the maximum value the structural + * size field can natively support. + */ extra_bytes = offsetof(typeof(*param), set_program_ex) + sizeof(param->set_program_ex); - max_size = ec->ec_dev->max_request - extra_bytes; + max_size = min_t(size_t, ec->ec_dev->max_request - extra_bytes, + type_max(typeof(param->set_program_ex.size))); } msg = alloc_lightbar_cmd_msg(ec); diff --git a/drivers/platform/chrome/cros_ec_sensorhub_ring.c b/drivers/platform/chrome/cros_ec_sensorhub_ring.c index e613dce244302f..d92b6021372073 100644 --- a/drivers/platform/chrome/cros_ec_sensorhub_ring.c +++ b/drivers/platform/chrome/cros_ec_sensorhub_ring.c @@ -836,8 +836,15 @@ static void cros_ec_sensorhub_ring_handler(struct cros_ec_sensorhub *sensorhub) sensorhub->msg->outsize = 1; sensorhub->msg->insize = fifo_info_length; - if (cros_ec_cmd_xfer_status(ec->ec_dev, sensorhub->msg) < 0) + ret = cros_ec_cmd_xfer_status(ec->ec_dev, sensorhub->msg); + if (ret < 0) + goto error; + if (ret != fifo_info_length) { + dev_warn_ratelimited(sensorhub->dev, + "Mismatch read length: size %d - expected %d\n", + ret, fifo_info_length); goto error; + } memcpy(fifo_info, &sensorhub->resp->fifo_info, fifo_info_length); diff --git a/drivers/platform/chrome/cros_ec_typec.c b/drivers/platform/chrome/cros_ec_typec.c index c0806c562bb931..50a68819ceb7bb 100644 --- a/drivers/platform/chrome/cros_ec_typec.c +++ b/drivers/platform/chrome/cros_ec_typec.c @@ -1119,6 +1119,12 @@ static void cros_typec_register_partner_pdos(struct cros_typec_data *typec, if (!resp->source_cap_count && !resp->sink_cap_count) return; + if (resp->source_cap_count > PDO_MAX_OBJECTS || + resp->sink_cap_count > PDO_MAX_OBJECTS) { + dev_warn(typec->dev, "Invalid PDO count from EC, port: %d\n", port_num); + return; + } + port->partner_pd = typec_partner_usb_power_delivery_register(port->partner, &desc); if (IS_ERR(port->partner_pd)) { dev_warn(typec->dev, "Failed to register partner PD device, port: %d\n", port_num); diff --git a/drivers/platform/mellanox/mlxbf-bootctl.c b/drivers/platform/mellanox/mlxbf-bootctl.c index f67c7f56ab2b70..9ddc7fa1a9732c 100644 --- a/drivers/platform/mellanox/mlxbf-bootctl.c +++ b/drivers/platform/mellanox/mlxbf-bootctl.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include diff --git a/drivers/platform/mellanox/mlxbf-pmc.c b/drivers/platform/mellanox/mlxbf-pmc.c index 5ec1ad4716967d..2ad9e2b0493c4b 100644 --- a/drivers/platform/mellanox/mlxbf-pmc.c +++ b/drivers/platform/mellanox/mlxbf-pmc.c @@ -2262,13 +2262,19 @@ static int mlxbf_pmc_map_counters(struct device *dev) static int mlxbf_pmc_probe(struct platform_device *pdev) { - struct acpi_device *acpi_dev = ACPI_COMPANION(&pdev->dev); - const char *hid = acpi_device_hid(acpi_dev); struct device *dev = &pdev->dev; + struct acpi_device *acpi_dev; struct arm_smccc_res res; + const char *hid; guid_t guid; int ret; + acpi_dev = ACPI_COMPANION(&pdev->dev); + if (!acpi_dev) + return -ENODEV; + + hid = acpi_device_hid(acpi_dev); + /* Ensure we have the UUID we expect for this service. */ arm_smccc_smc(MLXBF_PMC_SIP_SVC_UID, 0, 0, 0, 0, 0, 0, 0, &res); guid_parse(mlxbf_pmc_svc_uuid_str, &guid); diff --git a/drivers/platform/surface/surface_acpi_notify.c b/drivers/platform/surface/surface_acpi_notify.c index a9dcb0bbe90eee..593a7aba62432e 100644 --- a/drivers/platform/surface/surface_acpi_notify.c +++ b/drivers/platform/surface/surface_acpi_notify.c @@ -777,12 +777,16 @@ static int san_consumer_links_setup(struct platform_device *pdev) static int san_probe(struct platform_device *pdev) { - struct acpi_device *san = ACPI_COMPANION(&pdev->dev); struct ssam_controller *ctrl; + struct acpi_device *san; struct san_data *data; acpi_status astatus; int status; + san = ACPI_COMPANION(&pdev->dev); + if (!san) + return -ENODEV; + ctrl = ssam_client_bind(&pdev->dev); if (IS_ERR(ctrl)) return PTR_ERR(ctrl) == -ENODEV ? -EPROBE_DEFER : PTR_ERR(ctrl); diff --git a/drivers/platform/x86/acer-wmi.c b/drivers/platform/x86/acer-wmi.c index e0eaaefb13d04c..61ae622c93d949 100644 --- a/drivers/platform/x86/acer-wmi.c +++ b/drivers/platform/x86/acer-wmi.c @@ -1581,7 +1581,9 @@ static int WMI_gaming_execute_u32_u64(u32 method_id, u32 in, u64 *out) return -EIO; obj = result.pointer; - if (obj && out) { + if (!obj && out) { + ret = -ENOMSG; + } else if (obj && out) { switch (obj->type) { case ACPI_TYPE_INTEGER: *out = obj->integer.value; diff --git a/drivers/platform/x86/amd/hsmp/hwmon.c b/drivers/platform/x86/amd/hsmp/hwmon.c index 0cc9a742497fc3..c8314eee06f4f0 100644 --- a/drivers/platform/x86/amd/hsmp/hwmon.c +++ b/drivers/platform/x86/amd/hsmp/hwmon.c @@ -31,6 +31,9 @@ static int hsmp_hwmon_write(struct device *dev, enum hwmon_sensor_types type, if (attr != hwmon_power_cap) return -EOPNOTSUPP; + if (val < 0) + return -EINVAL; + msg.num_args = 1; msg.args[0] = val / MICROWATT_PER_MILLIWATT; msg.msg_id = HSMP_SET_SOCKET_POWER_LIMIT; diff --git a/drivers/platform/x86/asus-wireless.c b/drivers/platform/x86/asus-wireless.c index 2b494bf3cba8c9..aab45f0442c5ba 100644 --- a/drivers/platform/x86/asus-wireless.c +++ b/drivers/platform/x86/asus-wireless.c @@ -132,6 +132,10 @@ static int asus_wireless_probe(struct platform_device *pdev) const struct acpi_device_id *id; int err; + id = acpi_match_acpi_device(device_ids, adev); + if (!id) + return -ENODEV; + data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; @@ -139,6 +143,7 @@ static int asus_wireless_probe(struct platform_device *pdev) platform_set_drvdata(pdev, data); data->adev = adev; + data->hswc_params = (const struct hswc_params *)id->driver_data; data->idev = devm_input_allocate_device(&pdev->dev); if (!data->idev) @@ -153,12 +158,6 @@ static int asus_wireless_probe(struct platform_device *pdev) if (err) return err; - id = acpi_match_acpi_device(device_ids, adev); - if (!id) - return 0; - - data->hswc_params = (const struct hswc_params *)id->driver_data; - data->wq = create_singlethread_workqueue("asus_wireless_workqueue"); if (!data->wq) return -ENOMEM; diff --git a/drivers/platform/x86/asus-wmi.c b/drivers/platform/x86/asus-wmi.c index e835779b6f5f1a..c162fbdb5b1069 100644 --- a/drivers/platform/x86/asus-wmi.c +++ b/drivers/platform/x86/asus-wmi.c @@ -5244,20 +5244,20 @@ static int asus_wmi_add(struct platform_device *pdev) return 0; fail_wmi_handler: + asus_screenpad_exit(asus); +fail_screenpad: asus_wmi_backlight_exit(asus); fail_backlight: asus_wmi_rfkill_exit(asus); -fail_screenpad: - asus_screenpad_exit(asus); fail_rfkill: asus_wmi_led_exit(asus); fail_leds: +fail_custom_fan_curve: fail_hwmon: asus_wmi_input_exit(asus); fail_input: asus_wmi_sysfs_exit(asus->platform_device); fail_sysfs: -fail_custom_fan_curve: fail_platform_profile_setup: fail_fan_boost_mode: fail_platform: diff --git a/drivers/platform/x86/dell/dell-wmi-base.c b/drivers/platform/x86/dell/dell-wmi-base.c index 997383ba18465c..38a6b3ae2f75c1 100644 --- a/drivers/platform/x86/dell/dell-wmi-base.c +++ b/drivers/platform/x86/dell/dell-wmi-base.c @@ -456,7 +456,7 @@ static int dell_wmi_process_key(struct wmi_device *wdev, int type, int code, __l key++; used = 1; } else if (type == 0x0012 && code == 0x000d && remaining > 0) { - value = (le16_to_cpu(buffer[2]) == 2); + value = (le16_to_cpu(buffer[0]) == 2); used = 1; } @@ -843,9 +843,22 @@ static int __init dell_wmi_init(void) err = dell_privacy_register_driver(); if (err) - return err; + goto out_smbios; - return wmi_driver_register(&dell_wmi_driver); + err = wmi_driver_register(&dell_wmi_driver); + if (err) + goto out_privacy; + + return 0; + +out_privacy: + dell_privacy_unregister_driver(); + +out_smbios: + if (wmi_requires_smbios_request) + dell_wmi_events_set_enabled(false); + + return err; } late_initcall(dell_wmi_init); diff --git a/drivers/platform/x86/dell/dell-wmi-privacy.c b/drivers/platform/x86/dell/dell-wmi-privacy.c index f9d275b2f900c0..366e5b8dc868f7 100644 --- a/drivers/platform/x86/dell/dell-wmi-privacy.c +++ b/drivers/platform/x86/dell/dell-wmi-privacy.c @@ -92,11 +92,11 @@ bool dell_privacy_has_mic_mute(void) { struct privacy_wmi_data *priv; - mutex_lock(&list_mutex); + guard(mutex)(&list_mutex); + priv = list_first_entry_or_null(&wmi_list, struct privacy_wmi_data, list); - mutex_unlock(&list_mutex); return priv && (priv->features_present & BIT(DELL_PRIVACY_TYPE_AUDIO)); } diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h b/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h index 3bddedad5eba2a..eb48ced5582323 100644 --- a/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h +++ b/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h @@ -107,7 +107,7 @@ enum { static int get_##type##_instance_id(struct kobject *kobj) \ { \ int i; \ - for (i = 0; i <= wmi_priv.type##_instances_count; i++) { \ + for (i = 0; i < wmi_priv.type##_instances_count; i++) { \ if (!(strcmp(kobj->name, wmi_priv.type##_data[i].attribute_name)))\ return i; \ } \ diff --git a/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c index 9821403e42b644..a9e17863741622 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c @@ -353,6 +353,11 @@ static int hp_populate_password_elements_from_package(union acpi_object *passwor case PSWD_ENCODINGS: size = min_t(u32, password_data->encodings_size, MAX_ENCODINGS_SIZE); for (pos_values = 0; pos_values < size; pos_values++) { + if (elem + pos_values >= password_obj_count) { + pr_err("Error elem-objects package is too small\n"); + return -EINVAL; + } + ret = hp_convert_hexstr_to_str(password_obj[elem + pos_values].string.pointer, password_obj[elem + pos_values].string.length, &str_value, &value_len); diff --git a/drivers/platform/x86/lg-laptop.c b/drivers/platform/x86/lg-laptop.c index a8f2f465ef3f96..a2db9657027e58 100644 --- a/drivers/platform/x86/lg-laptop.c +++ b/drivers/platform/x86/lg-laptop.c @@ -100,6 +100,7 @@ static u32 inited; #define INIT_SPARSE_KEYMAP 0x80 static int battery_limit_use_wmbb; +static bool kbd_backlight_available; static struct led_classdev kbd_backlight; static enum led_brightness get_kbd_backlight_level(struct device *dev); @@ -214,6 +215,7 @@ static union acpi_object *lg_wmbb(struct device *dev, u32 method_id, u32 arg1, u static void wmi_notify(union acpi_object *obj, void *context) { long data = (long)context; + unsigned int brightness; pr_debug("event guid %li\n", data); if (!obj) @@ -224,8 +226,11 @@ static void wmi_notify(union acpi_object *obj, void *context) struct key_entry *key; if (eventcode == 0x10000000) { - led_classdev_notify_brightness_hw_changed( - &kbd_backlight, get_kbd_backlight_level(kbd_backlight.dev->parent)); + if (kbd_backlight_available) { + brightness = get_kbd_backlight_level(kbd_backlight.dev->parent); + led_classdev_notify_brightness_hw_changed(&kbd_backlight, + brightness); + } } else { key = sparse_keymap_entry_from_scancode( wmi_input_dev, eventcode); @@ -865,8 +870,13 @@ static int acpi_probe(struct platform_device *pdev) goto out_platform_device; /* LEDs are optional */ - led_classdev_register(&pf_device->dev, &kbd_backlight); - led_classdev_register(&pf_device->dev, &tpad_led); + ret = devm_led_classdev_register(&pdev->dev, &kbd_backlight); + if (ret < 0) + kbd_backlight_available = false; + else + kbd_backlight_available = true; + + devm_led_classdev_register(&pdev->dev, &tpad_led); wmi_input_setup(); battery_hook_register(&battery_hook); @@ -884,9 +894,6 @@ static void acpi_remove(struct platform_device *pdev) { sysfs_remove_group(&pf_device->dev.kobj, &dev_attribute_group); - led_classdev_unregister(&tpad_led); - led_classdev_unregister(&kbd_backlight); - battery_hook_unregister(&battery_hook); wmi_input_destroy(); platform_device_unregister(pf_device); diff --git a/drivers/pmdomain/bcm/bcm2835-power.c b/drivers/pmdomain/bcm/bcm2835-power.c index b76d74e3849be0..68a0a7a8cee381 100644 --- a/drivers/pmdomain/bcm/bcm2835-power.c +++ b/drivers/pmdomain/bcm/bcm2835-power.c @@ -677,7 +677,12 @@ static int bcm2835_power_probe(struct platform_device *pdev) if (ret) goto fail; - of_genpd_add_provider_onecell(dev->parent->of_node, &power->pd_xlate); + ret = of_genpd_add_provider_onecell(dev->parent->of_node, + &power->pd_xlate); + if (ret) { + dev_err_probe(dev, ret, "failed to add genpd provider\n"); + goto fail; + } dev_info(dev, "Broadcom BCM2835 power domains driver"); return 0; diff --git a/drivers/power/reset/reboot-mode.c b/drivers/power/reset/reboot-mode.c index d20e44db053256..af00c00eceee7f 100644 --- a/drivers/power/reset/reboot-mode.c +++ b/drivers/power/reset/reboot-mode.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -168,10 +169,11 @@ error: */ int reboot_mode_register(struct reboot_mode_driver *reboot) { - struct mode_info *info; + struct mode_info *info = NULL; struct property *prop; struct device_node *np = reboot->dev->of_node; size_t len = strlen(PREFIX); + u32 magic; int ret; INIT_LIST_HEAD(&reboot->head); @@ -180,22 +182,22 @@ int reboot_mode_register(struct reboot_mode_driver *reboot) if (strncmp(prop->name, PREFIX, len)) continue; - info = devm_kzalloc(reboot->dev, sizeof(*info), GFP_KERNEL); + if (device_property_read_u32(reboot->dev, prop->name, &magic)) { + dev_dbg(reboot->dev, "reboot mode %s without magic number\n", + prop->name); + continue; + } + + info = kzalloc_obj(*info, GFP_KERNEL); if (!info) { ret = -ENOMEM; goto error; } - if (of_property_read_u32(np, prop->name, &info->magic)) { - dev_err(reboot->dev, "reboot mode %s without magic number\n", - info->mode); - devm_kfree(reboot->dev, info); - continue; - } - + info->magic = magic; info->mode = kstrdup_const(prop->name + len, GFP_KERNEL); if (!info->mode) { - ret = -ENOMEM; + ret = -ENOMEM; goto error; } else if (info->mode[0] == '\0') { kfree_const(info->mode); @@ -206,6 +208,7 @@ int reboot_mode_register(struct reboot_mode_driver *reboot) } list_add_tail(&info->list, &reboot->head); + info = NULL; } reboot->reboot_notifier.notifier_call = reboot_mode_notify; @@ -218,6 +221,7 @@ int reboot_mode_register(struct reboot_mode_driver *reboot) return 0; error: + kfree(info); reboot_mode_unregister(reboot); return ret; } @@ -261,12 +265,16 @@ static inline void reboot_mode_unregister_device(struct reboot_mode_driver *rebo int reboot_mode_unregister(struct reboot_mode_driver *reboot) { struct mode_info *info; + struct mode_info *next; unregister_reboot_notifier(&reboot->reboot_notifier); reboot_mode_unregister_device(reboot); - list_for_each_entry(info, &reboot->head, list) + list_for_each_entry_safe(info, next, &reboot->head, list) { + list_del(&info->list); kfree_const(info->mode); + kfree(info); + } return 0; } diff --git a/drivers/power/supply/bd71828-power.c b/drivers/power/supply/bd71828-power.c index 19f24f859666a0..3be7a75a7d6ca3 100644 --- a/drivers/power/supply/bd71828-power.c +++ b/drivers/power/supply/bd71828-power.c @@ -191,12 +191,13 @@ static int bd71828_get_current_ds_adc(struct bd71828_power *pwr, int *curr, int { __be16 tmp_curr; char *tmp = (char *)&tmp_curr; - int dir = 1; int regs[] = { pwr->regs->ibat, pwr->regs->ibat_avg }; int *vals[] = { curr, curr_avg }; int ret, i; - for (dir = 1, i = 0; i < ARRAY_SIZE(regs); i++) { + for (i = 0; i < ARRAY_SIZE(regs); i++) { + int dir = 1; + ret = regmap_bulk_read(pwr->regmap, regs[i], &tmp_curr, sizeof(tmp_curr)); if (ret) @@ -224,13 +225,13 @@ static int bd71815_get_temp(struct bd71828_power *pwr, int *temp) if (ret) return ret; - t = 200 - t; - if (t > 200) { dev_err(pwr->dev, "Failed to read battery temperature\n"); return -ENODATA; } + *temp = 200 - t; + return 0; } @@ -454,8 +455,10 @@ static int bd71828_charger_get_property(struct power_supply *psy, switch (psp) { case POWER_SUPPLY_PROP_ONLINE: ret = get_chg_online(pwr, &online); - if (!ret) - val->intval = online; + if (ret) + return ret; + + val->intval = online; break; case POWER_SUPPLY_PROP_VOLTAGE_NOW: ret = bd7182x_read16_himask(pwr, pwr->regs->vdcin, @@ -638,7 +641,6 @@ static const enum power_supply_property bd71828_charger_props[] = { static const enum power_supply_property bd71828_battery_props[] = { POWER_SUPPLY_PROP_STATUS, - POWER_SUPPLY_PROP_HEALTH, POWER_SUPPLY_PROP_VOLTAGE_NOW, POWER_SUPPLY_PROP_HEALTH, POWER_SUPPLY_PROP_PRESENT, @@ -1217,7 +1219,9 @@ static int bd71828_power_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, ret, "sense resistor missing\n"); dev_set_drvdata(&pdev->dev, pwr); - bd71828_init_hardware(pwr); + ret = bd71828_init_hardware(pwr); + if (ret) + return dev_err_probe(&pdev->dev, ret, "hardware init failed\n"); bat_cfg.drv_data = pwr; bat_cfg.fwnode = dev_fwnode(&pdev->dev); diff --git a/drivers/power/supply/bd99954-charger.h b/drivers/power/supply/bd99954-charger.h index f588979253830d..d053ac901274c7 100644 --- a/drivers/power/supply/bd99954-charger.h +++ b/drivers/power/supply/bd99954-charger.h @@ -279,17 +279,7 @@ enum bd9995x_fields { F_VBUS_EXTID, F_VBUS_IDRDET, F_VBUS_INDO, - F_VBUS_UCDSWEN, - F_VBUS_RREF_EN, - F_VBUS_DPPU_EN, - F_VBUS_DPREF_EN, - F_VBUS_DMREF_EN, - F_VBUS_DPDET_EN, - F_VBUS_DMDET_EN, - F_VBUS_DPSINK_EN, - F_VBUS_DMSINK_EN, - F_VBUS_DP_BUFF_EN, - F_VBUS_DM_BUFF_EN, + F_VBUS_EXTCLKENBL, F_VBUS_PLSTESTEN, F_VBUS_UCDSWEN_TSTENB, @@ -626,17 +616,6 @@ static const struct reg_field bd9995x_reg_fields[] = { [F_VBUS_EXTID] = REG_FIELD(VBUS_IDD_STATUS, 5, 5), [F_VBUS_IDRDET] = REG_FIELD(VBUS_IDD_STATUS, 4, 4), [F_VBUS_INDO] = REG_FIELD(VBUS_IDD_STATUS, 0, 3), - [F_VBUS_UCDSWEN] = REG_FIELD(VCC_UCD_FCTRL_SET, 10, 10), - [F_VBUS_RREF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 9, 9), - [F_VBUS_DPPU_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 8, 8), - [F_VBUS_DPREF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 7, 7), - [F_VBUS_DMREF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 6, 6), - [F_VBUS_DPDET_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 5, 5), - [F_VBUS_DMDET_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 4, 4), - [F_VBUS_DPSINK_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 3, 3), - [F_VBUS_DMSINK_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 2, 2), - [F_VBUS_DP_BUFF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 1, 1), - [F_VBUS_DM_BUFF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 0, 0), [F_VBUS_EXTCLKENBL] = REG_FIELD(VBUS_UCD_FCTRL_EN, 15, 15), [F_VBUS_PLSTESTEN] = REG_FIELD(VBUS_UCD_FCTRL_EN, 14, 14), diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index 45f0e39b8c2dd4..01c9ec1116fa09 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -330,7 +330,7 @@ static u8 [BQ27XXX_REG_FLAGS] = 0x0a, [BQ27XXX_REG_TTE] = 0x16, [BQ27XXX_REG_TTF] = INVALID_REG_ADDR, - [BQ27XXX_REG_TTES] = 0x1c, + [BQ27XXX_REG_TTES] = 0x1a, [BQ27XXX_REG_TTECP] = INVALID_REG_ADDR, [BQ27XXX_REG_NAC] = 0x0c, [BQ27XXX_REG_RC] = 0x10, @@ -495,7 +495,7 @@ static u8 [BQ27XXX_REG_RC] = 0x10, [BQ27XXX_REG_FCC] = 0x12, [BQ27XXX_REG_CYCT] = 0x2a, - [BQ27XXX_REG_AE] = 0x22, + [BQ27XXX_REG_AE] = INVALID_REG_ADDR, [BQ27XXX_REG_SOC] = 0x2c, [BQ27XXX_REG_DCAP] = 0x3c, [BQ27XXX_REG_AP] = 0x22, @@ -516,7 +516,7 @@ static u8 [BQ27XXX_REG_RC] = 0x10, [BQ27XXX_REG_FCC] = 0x12, [BQ27XXX_REG_CYCT] = 0x2a, - [BQ27XXX_REG_AE] = 0x22, + [BQ27XXX_REG_AE] = INVALID_REG_ADDR, [BQ27XXX_REG_SOC] = 0x2c, [BQ27XXX_REG_DCAP] = 0x3c, [BQ27XXX_REG_AP] = 0x22, diff --git a/drivers/power/supply/isp1704_charger.c b/drivers/power/supply/isp1704_charger.c index 237912a9227245..e329321d06dbdd 100644 --- a/drivers/power/supply/isp1704_charger.c +++ b/drivers/power/supply/isp1704_charger.c @@ -482,6 +482,7 @@ static void isp1704_charger_remove(struct platform_device *pdev) struct isp1704_charger *isp = platform_get_drvdata(pdev); usb_unregister_notifier(isp->phy, &isp->nb); + cancel_work_sync(&isp->work); power_supply_unregister(isp->psy); isp1704_charger_set_power(isp, 0); } diff --git a/drivers/power/supply/sbs-battery.c b/drivers/power/supply/sbs-battery.c index 017ec06be76668..9bdb6c599c5f63 100644 --- a/drivers/power/supply/sbs-battery.c +++ b/drivers/power/supply/sbs-battery.c @@ -217,6 +217,7 @@ struct sbs_info { u32 flags; int technology; char strings[NR_STRING_BUFFERS][I2C_SMBUS_BLOCK_MAX + 1]; + char serial[5]; }; static char *sbs_get_string_buf(struct sbs_info *chip, @@ -821,18 +822,18 @@ static int sbs_get_battery_capacity(struct i2c_client *client, return 0; } -static char sbs_serial[5]; static int sbs_get_battery_serial_number(struct i2c_client *client, union power_supply_propval *val) { + struct sbs_info *chip = i2c_get_clientdata(client); int ret; ret = sbs_read_word_data(client, sbs_data[REG_SERIAL_NUMBER].addr); if (ret < 0) return ret; - sprintf(sbs_serial, "%04x", ret); - val->strval = sbs_serial; + sprintf(chip->serial, "%04x", ret); + val->strval = chip->serial; return 0; } diff --git a/drivers/power/supply/sc2731_charger.c b/drivers/power/supply/sc2731_charger.c index 58b86fd7877136..2b25e44da7978b 100644 --- a/drivers/power/supply/sc2731_charger.c +++ b/drivers/power/supply/sc2731_charger.c @@ -466,6 +466,7 @@ static int sc2731_charger_probe(struct platform_device *pdev) mutex_init(&info->lock); info->dev = &pdev->dev; INIT_WORK(&info->work, sc2731_charger_work); + platform_set_drvdata(pdev, info); info->regmap = dev_get_regmap(pdev->dev.parent, NULL); if (!info->regmap) { @@ -516,6 +517,7 @@ static void sc2731_charger_remove(struct platform_device *pdev) struct sc2731_charger_info *info = platform_get_drvdata(pdev); usb_unregister_notifier(info->usb_phy, &info->usb_notify); + cancel_work_sync(&info->work); } static const struct of_device_id sc2731_charger_of_match[] = { diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 1006d183d5081a..6b7d11a0abc5d0 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -32,6 +32,9 @@ #define ENERGY_STATUS_MASK GENMASK(31, 0) +/* Width of the RAPL energy counters, see the *_ENERGY_STATUS_MASK defines */ +#define RAPL_CNTR_WIDTH 32 + #define POWER_UNIT_OFFSET 0x00 #define POWER_UNIT_MASK GENMASK(3, 0) @@ -1227,6 +1230,7 @@ static u64 rapl_event_update(struct perf_event *event) struct rapl_package_pmu_data *data = event_to_pmu_data(event); u64 prev_raw_count, new_raw_count; s64 delta, sdelta; + int shift = 64 - RAPL_CNTR_WIDTH; /* * Follow the generic code to drain hwc->prev_count. @@ -1243,8 +1247,13 @@ static u64 rapl_event_update(struct perf_event *event) * Now we have the new raw value and have updated the prev * timestamp already. We can now calculate the elapsed delta * (event-)time and add that to the generic event. + * + * Careful, the counter is narrower than u64 and is not + * sign-extended above its physical width. Shift both values up + * so that the subtraction wraps, then shift the result back down. */ - delta = new_raw_count - prev_raw_count; + delta = (new_raw_count << shift) - (prev_raw_count << shift); + delta >>= shift; /* * Scale delta to smallest unit (2^-32) diff --git a/drivers/powercap/intel_rapl_tpmi.c b/drivers/powercap/intel_rapl_tpmi.c index 7f41491d9cd115..73f36d9c09b1b6 100644 --- a/drivers/powercap/intel_rapl_tpmi.c +++ b/drivers/powercap/intel_rapl_tpmi.c @@ -414,7 +414,10 @@ static int intel_rapl_tpmi_probe(struct auxiliary_device *auxdev, goto err; } - rapl_package_add_pmu(trp->rp); + ret = rapl_package_add_pmu(trp->rp); + if (ret) + dev_info(&auxdev->dev, "Failed to add RAPL PMU for Package%d, %d\n", + info->package_id, ret); auxiliary_set_drvdata(auxdev, trp); diff --git a/drivers/pps/clients/pps-gpio.c b/drivers/pps/clients/pps-gpio.c index 402f910f3e25db..73ec2c7335e518 100644 --- a/drivers/pps/clients/pps-gpio.c +++ b/drivers/pps/clients/pps-gpio.c @@ -31,38 +31,37 @@ struct pps_gpio_device_data { struct gpio_desc *echo_pin; struct timer_list echo_timer; /* timer to reset echo active state */ bool assert_falling_edge; - bool capture_clear; unsigned int echo_active_ms; /* PPS echo active duration */ unsigned long echo_timeout; /* timer timeout value in jiffies */ + struct pps_event_time ts; /* timestamp captured in hardirq */ }; /* * Report the PPS event */ -static irqreturn_t pps_gpio_irq_handler(int irq, void *data) +/* + * Primary hardirq handler -- runs in hardirq context even on PREEMPT_RT. + * Only captures the timestamp; all other work is deferred to the thread. + */ +static irqreturn_t pps_gpio_irq_hardirq(int irq, void *data) { - const struct pps_gpio_device_data *info; - struct pps_event_time ts; - int rising_edge; - - /* Get the time stamp first */ - pps_get_ts(&ts); - - info = data; - - /* Small trick to bypass the check on edge's direction when capture_clear is unset */ - rising_edge = info->capture_clear ? - gpiod_get_value(info->gpio_pin) : !info->assert_falling_edge; - if ((rising_edge && !info->assert_falling_edge) || - (!rising_edge && info->assert_falling_edge)) - pps_event(info->pps, &ts, PPS_CAPTUREASSERT, data); - else if (info->capture_clear && - ((rising_edge && info->assert_falling_edge) || - (!rising_edge && !info->assert_falling_edge))) - pps_event(info->pps, &ts, PPS_CAPTURECLEAR, data); - else - dev_warn_ratelimited(&info->pps->dev, "IRQ did not trigger any PPS event\n"); + struct pps_gpio_device_data *info = data; + + pps_get_ts(&info->ts); + + return IRQ_WAKE_THREAD; +} + +/* + * Threaded handler -- processes the PPS event using the timestamp + * captured in hardirq context above. + */ +static irqreturn_t pps_gpio_irq_thread(int irq, void *data) +{ + struct pps_gpio_device_data *info = data; + + pps_event(info->pps, &info->ts, PPS_CAPTUREASSERT, data); return IRQ_HANDLED; } @@ -78,11 +77,6 @@ static void pps_gpio_echo(struct pps_device *pps, int event, void *data) if (pps->params.mode & PPS_ECHOASSERT) gpiod_set_value(info->echo_pin, 1); break; - - case PPS_CAPTURECLEAR: - if (pps->params.mode & PPS_ECHOCLEAR) - gpiod_set_value(info->echo_pin, 1); - break; } /* fire the timer */ @@ -144,15 +138,8 @@ static int pps_gpio_setup(struct device *dev) static unsigned long get_irqf_trigger_flags(const struct pps_gpio_device_data *data) { - unsigned long flags = data->assert_falling_edge ? - IRQF_TRIGGER_FALLING : IRQF_TRIGGER_RISING; - - if (data->capture_clear) { - flags |= ((flags & IRQF_TRIGGER_RISING) ? - IRQF_TRIGGER_FALLING : IRQF_TRIGGER_RISING); - } - - return flags; + return data->assert_falling_edge ? IRQF_TRIGGER_FALLING : + IRQF_TRIGGER_RISING; } static int pps_gpio_probe(struct platform_device *pdev) @@ -185,9 +172,6 @@ static int pps_gpio_probe(struct platform_device *pdev) /* initialize PPS specific parts of the bookkeeping data structure. */ data->info.mode = PPS_CAPTUREASSERT | PPS_OFFSETASSERT | PPS_ECHOASSERT | PPS_CANWAIT | PPS_TSFMT_TSPEC; - if (data->capture_clear) - data->info.mode |= PPS_CAPTURECLEAR | PPS_OFFSETCLEAR | - PPS_ECHOCLEAR; data->info.owner = THIS_MODULE; snprintf(data->info.name, PPS_MAX_NAME_LEN - 1, "%s.%d", pdev->name, pdev->id); @@ -199,8 +183,6 @@ static int pps_gpio_probe(struct platform_device *pdev) /* register PPS source */ pps_default_params = PPS_CAPTUREASSERT | PPS_OFFSETASSERT; - if (data->capture_clear) - pps_default_params |= PPS_CAPTURECLEAR | PPS_OFFSETCLEAR; data->pps = pps_register_source(&data->info, pps_default_params); if (IS_ERR(data->pps)) { dev_err(dev, "failed to register IRQ %d as PPS source\n", @@ -209,8 +191,10 @@ static int pps_gpio_probe(struct platform_device *pdev) } /* register IRQ interrupt handler */ - ret = request_irq(data->irq, pps_gpio_irq_handler, - get_irqf_trigger_flags(data), data->info.name, data); + ret = request_threaded_irq(data->irq, + pps_gpio_irq_hardirq, pps_gpio_irq_thread, + get_irqf_trigger_flags(data) | IRQF_ONESHOT, + data->info.name, data); if (ret) { pps_unregister_source(data->pps); dev_err(dev, "failed to acquire IRQ %d\n", data->irq); diff --git a/drivers/pps/pps.c b/drivers/pps/pps.c index de1122bb69ea64..7ecdd774a44b90 100644 --- a/drivers/pps/pps.c +++ b/drivers/pps/pps.c @@ -66,13 +66,19 @@ static int pps_cdev_pps_fetch(struct pps_device *pps, struct pps_fdata *fdata) err = wait_event_interruptible(pps->queue, ev != pps->last_ev); else { + struct timespec64 ts; unsigned long ticks; dev_dbg(&pps->dev, "timeout %lld.%09d\n", (long long) fdata->timeout.sec, fdata->timeout.nsec); - ticks = fdata->timeout.sec * HZ; - ticks += fdata->timeout.nsec / (NSEC_PER_SEC / HZ); + + if (fdata->timeout.sec < 0) + return -ETIMEDOUT; + + ts.tv_sec = fdata->timeout.sec; + ts.tv_nsec = fdata->timeout.nsec; + ticks = timespec64_to_jiffies(&ts); if (ticks != 0) { err = wait_event_interruptible_timeout( diff --git a/drivers/ptp/ptp_netc.c b/drivers/ptp/ptp_netc.c index 1c20d7efab9293..19ca99d80e95b5 100644 --- a/drivers/ptp/ptp_netc.c +++ b/drivers/ptp/ptp_netc.c @@ -440,7 +440,7 @@ static int net_timer_enable_perout(struct netc_timer *priv, } if (on) { - u64 period_ns, gclk_period, max_period, min_period; + u64 period_ns, gclk_period, min_period; struct timespec64 period, stime; u32 integral_period; int alarm_id; @@ -450,12 +450,12 @@ static int net_timer_enable_perout(struct netc_timer *priv, period_ns = timespec64_to_ns(&period); integral_period = netc_timer_get_integral_period(priv); - max_period = (u64)NETC_TMR_DEFAULT_FIPER + integral_period; gclk_period = netc_timer_get_gclk_period(priv); min_period = gclk_period * 4 + integral_period; - if (period_ns > max_period || period_ns < min_period) { - dev_err(dev, "The period range is %llu ~ %llu\n", - min_period, max_period); + if (period_ns > NETC_TMR_DEFAULT_FIPER || + period_ns < min_period) { + dev_err(dev, "The period range is %llu ~ %lu\n", + min_period, NETC_TMR_DEFAULT_FIPER); err = -EINVAL; goto unlock_spinlock; } @@ -482,6 +482,9 @@ static int net_timer_enable_perout(struct netc_timer *priv, netc_timer_enable_periodic_pulse(priv, channel); } else { + if (!pp->enabled) + goto unlock_spinlock; + netc_timer_disable_periodic_pulse(priv, channel); priv->fs_alarm_bitmap &= ~BIT(pp->alarm_id); memset(pp, 0, sizeof(*pp)); diff --git a/drivers/pwm/pwm-samsung.c b/drivers/pwm/pwm-samsung.c index 951b38ff5f8ecb..14fb460a45655f 100644 --- a/drivers/pwm/pwm-samsung.c +++ b/drivers/pwm/pwm-samsung.c @@ -102,7 +102,7 @@ struct samsung_pwm_chip { * IP. Should this change, both drivers will need to be modified to * properly synchronize accesses to particular instances. */ -static DEFINE_SPINLOCK(samsung_pwm_lock); +static DEFINE_RAW_SPINLOCK(samsung_pwm_lock); #endif static inline @@ -141,14 +141,14 @@ static void pwm_samsung_set_divisor(struct samsung_pwm_chip *our_chip, bits = (fls(divisor) - 1) - our_chip->variant.div_base; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); reg = readl(our_chip->base + REG_TCFG1); reg &= ~(TCFG1_MUX_MASK << shift); reg |= bits << shift; writel(reg, our_chip->base + REG_TCFG1); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); } static int pwm_samsung_is_tdiv(struct samsung_pwm_chip *our_chip, unsigned int chan) @@ -249,7 +249,7 @@ static int pwm_samsung_enable(struct pwm_chip *chip, struct pwm_device *pwm) unsigned long flags; u32 tcon; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); tcon = readl(our_chip->base + REG_TCON); @@ -263,7 +263,7 @@ static int pwm_samsung_enable(struct pwm_chip *chip, struct pwm_device *pwm) our_chip->disabled_mask &= ~BIT(pwm->hwpwm); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); return 0; } @@ -275,7 +275,7 @@ static void pwm_samsung_disable(struct pwm_chip *chip, struct pwm_device *pwm) unsigned long flags; u32 tcon; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); tcon = readl(our_chip->base + REG_TCON); tcon &= ~TCON_AUTORELOAD(tcon_chan); @@ -290,7 +290,7 @@ static void pwm_samsung_disable(struct pwm_chip *chip, struct pwm_device *pwm) our_chip->disabled_mask |= BIT(pwm->hwpwm); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); } static void pwm_samsung_manual_update(struct samsung_pwm_chip *our_chip, @@ -298,11 +298,11 @@ static void pwm_samsung_manual_update(struct samsung_pwm_chip *our_chip, { unsigned long flags; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); __pwm_samsung_manual_update(our_chip, pwm); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); } static int __pwm_samsung_config(struct pwm_chip *chip, struct pwm_device *pwm, @@ -390,7 +390,7 @@ static void pwm_samsung_set_invert(struct samsung_pwm_chip *our_chip, unsigned long flags; u32 tcon; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); tcon = readl(our_chip->base + REG_TCON); @@ -404,7 +404,7 @@ static void pwm_samsung_set_invert(struct samsung_pwm_chip *our_chip, writel(tcon, our_chip->base + REG_TCON); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); } static int pwm_samsung_set_polarity(struct pwm_chip *chip, diff --git a/drivers/rapidio/rio-scan.c b/drivers/rapidio/rio-scan.c index dcd6619a4b0277..3cc25d0534513c 100644 --- a/drivers/rapidio/rio-scan.c +++ b/drivers/rapidio/rio-scan.c @@ -874,6 +874,7 @@ static struct rio_net *rio_scan_alloc_net(struct rio_mport *mport, net->dev.release = rio_scan_release_dev; if (rio_add_net(net)) { put_device(&net->dev); + mport->net = NULL; net = NULL; } } diff --git a/drivers/regulator/adp5055-regulator.c b/drivers/regulator/adp5055-regulator.c index 9ebd52b3923573..7eaa7d9dc08cc0 100644 --- a/drivers/regulator/adp5055-regulator.c +++ b/drivers/regulator/adp5055-regulator.c @@ -224,7 +224,7 @@ static int adp5055_of_parse_cb(struct device_node *np, adp5055->dvs_limit_upper[id] = pval; if (adp5055->dvs_limit_upper[id] > 192000 || adp5055->dvs_limit_upper[id] < 12000) - return dev_err_probe(config->dev, adp5055->dvs_limit_upper[id], + return dev_err_probe(config->dev, -EINVAL, "Out of range - dvs-limit-upper-microvolt value."); ret = of_property_read_u32(np, "adi,dvs-limit-lower-microvolt", &pval); diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c index 2e61606fc1d057..6a4008f387b5e7 100644 --- a/drivers/regulator/core.c +++ b/drivers/regulator/core.c @@ -27,6 +27,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -6899,8 +6900,9 @@ static int __init regulator_init_complete(void) * we'd only do this on systems that need it, and a kernel * command line option might be useful. */ - schedule_delayed_work(®ulator_init_complete_work, - msecs_to_jiffies(30000)); + queue_delayed_work(system_freezable_wq, + ®ulator_init_complete_work, + msecs_to_jiffies(30000)); return 0; } diff --git a/drivers/regulator/qcom-rpmh-regulator.c b/drivers/regulator/qcom-rpmh-regulator.c index 73856e3df337ad..ef5523ff427b85 100644 --- a/drivers/regulator/qcom-rpmh-regulator.c +++ b/drivers/regulator/qcom-rpmh-regulator.c @@ -111,6 +111,7 @@ static const struct resource_name_formats vreg_rsc_name_lookup[NUM_REGULATOR_TYP * @hpm_min_load_uA: Minimum load current in microamps that requires * high power mode (HPM) operation. This is used * for LDO hardware type regulators only. + * @pmic_bypass_mode: The PMIC bypass mode value. * @pmic_mode_map: Array indexed by regulator framework mode * containing PMIC hardware modes. Must be large * enough to index all framework modes supported @@ -125,6 +126,7 @@ struct rpmh_vreg_hw_data { int n_linear_ranges; int n_voltages; int hpm_min_load_uA; + int pmic_bypass_mode; const int *pmic_mode_map; unsigned int (*of_map_mode)(unsigned int mode); }; @@ -311,7 +313,7 @@ static int rpmh_regulator_vrm_set_mode_bypass(struct rpmh_vreg *vreg, return pmic_mode; if (bypassed) - cmd.data = PMIC4_BOB_MODE_PASS; + cmd.data = vreg->hw_data->pmic_bypass_mode; else cmd.data = pmic_mode; @@ -767,6 +769,7 @@ static const struct rpmh_vreg_hw_data pmic4_bob = { }, .n_linear_ranges = 1, .n_voltages = 84, + .pmic_bypass_mode = PMIC4_BOB_MODE_PASS, .pmic_mode_map = pmic_mode_map_pmic4_bob, .of_map_mode = rpmh_regulator_pmic4_bob_of_map_mode, }; @@ -975,6 +978,7 @@ static const struct rpmh_vreg_hw_data pmic5_bob = { }, .n_linear_ranges = 1, .n_voltages = 32, + .pmic_bypass_mode = PMIC5_BOB_MODE_PASS, .pmic_mode_map = pmic_mode_map_pmic5_bob, .of_map_mode = rpmh_regulator_pmic4_bob_of_map_mode, }; diff --git a/drivers/regulator/tps65185.c b/drivers/regulator/tps65185.c index 786622d8d5980c..6a3130177e7a3f 100644 --- a/drivers/regulator/tps65185.c +++ b/drivers/regulator/tps65185.c @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -183,7 +184,10 @@ static int tps65185_vposneg_enable(struct regulator_dev *rdev) wait_for_completion_timeout(&data->pgood_completion, msecs_to_jiffies(PGOOD_TIMEOUT_MSECS)); dev_dbg(data->dev, "turned on"); - if (gpiod_get_value_cansleep(data->pgood_gpio) != 1) + ret = gpiod_get_value_cansleep(data->pgood_gpio); + if (ret < 0) + return ret; + if (!ret) return -ETIMEDOUT; return 0; @@ -372,6 +376,11 @@ static int tps65185_probe(struct i2c_client *client) return dev_err_probe(&client->dev, ret, "failed to get vin regulator\n"); + // TPS65185x PMIC for E Ink Vizplex Enabled Electronic Paper Display Chapter 7.6 Figure 2: + // "Minimum delay time between WAKEUP rising edge and IC ready to accept I2C transaction." + // https://www.ti.com/lit/ds/symlink/tps65185.pdf + usleep_range(1800, 3000); + data->dev = &client->dev; i2c_set_clientdata(client, data); diff --git a/drivers/regulator/tps6594-regulator.c b/drivers/regulator/tps6594-regulator.c index 645e83462c645e..31a5218d551059 100644 --- a/drivers/regulator/tps6594-regulator.c +++ b/drivers/regulator/tps6594-regulator.c @@ -669,13 +669,20 @@ static int tps6594_regulator_probe(struct platform_device *pdev) * buck_configured to avoid creating bucks for every buck in multiphase */ for (multi = 0; multi < desc->num_multi_phase_regs; multi++) { + struct device_node *np_parent; + multi_regs = &desc->multi_phase_regs[multi]; np = of_find_node_by_name(tps->dev->of_node, multi_regs->supply_name); - npname = of_node_full_name(np); - np_pmic_parent = of_get_parent(of_get_parent(np)); - if (of_node_cmp(of_node_full_name(np_pmic_parent), tps->dev->of_node->full_name)) + if (!np) continue; - if (strcmp(npname, multi_regs->supply_name) == 0) { + + npname = of_node_full_name(np); + np_parent = of_get_parent(np); + np_pmic_parent = of_get_parent(np_parent); + + if (np_pmic_parent && + !of_node_cmp(of_node_full_name(np_pmic_parent), tps->dev->of_node->full_name) && + strcmp(npname, multi_regs->supply_name) == 0) { switch (multi) { case MULTI_BUCK12: buck_multi[0] = true; @@ -706,6 +713,10 @@ static int tps6594_regulator_probe(struct platform_device *pdev) break; } } + + of_node_put(np_pmic_parent); + of_node_put(np_parent); + of_node_put(np); } reg_irq_nb = desc->num_irq_types * (desc->num_buck_regs + desc->num_ldo_regs); diff --git a/drivers/remoteproc/qcom_common.c b/drivers/remoteproc/qcom_common.c index e1a955476c9b3a..5294e327f1587e 100644 --- a/drivers/remoteproc/qcom_common.c +++ b/drivers/remoteproc/qcom_common.c @@ -253,8 +253,10 @@ void qcom_add_glink_subdev(struct rproc *rproc, struct qcom_rproc_glink *glink, return; glink->ssr_name = kstrdup_const(ssr_name, GFP_KERNEL); - if (!glink->ssr_name) + if (!glink->ssr_name) { + of_node_put(glink->node); return; + } glink->dev = dev; glink->subdev.start = glink_subdev_start; diff --git a/drivers/remoteproc/qcom_common.h b/drivers/remoteproc/qcom_common.h index b07fbaa091a064..b0e7e336d363ec 100644 --- a/drivers/remoteproc/qcom_common.h +++ b/drivers/remoteproc/qcom_common.h @@ -68,6 +68,7 @@ struct qcom_sysmon *qcom_add_sysmon_subdev(struct rproc *rproc, int ssctl_instance); void qcom_remove_sysmon_subdev(struct qcom_sysmon *sysmon); bool qcom_sysmon_shutdown_acked(struct qcom_sysmon *sysmon); +bool qcom_sysmon_shutdown_irq_state(struct qcom_sysmon *sysmon); #else static inline struct qcom_sysmon *qcom_add_sysmon_subdev(struct rproc *rproc, const char *name, @@ -84,6 +85,11 @@ static inline bool qcom_sysmon_shutdown_acked(struct qcom_sysmon *sysmon) { return false; } + +static inline bool qcom_sysmon_shutdown_irq_state(struct qcom_sysmon *sysmon) +{ + return false; +} #endif #endif diff --git a/drivers/remoteproc/qcom_q6v5.c b/drivers/remoteproc/qcom_q6v5.c index 58d5b85e58cdad..241478ea29782a 100644 --- a/drivers/remoteproc/qcom_q6v5.c +++ b/drivers/remoteproc/qcom_q6v5.c @@ -197,12 +197,13 @@ static irqreturn_t q6v5_stop_interrupt(int irq, void *data) */ int qcom_q6v5_request_stop(struct qcom_q6v5 *q6v5, struct qcom_sysmon *sysmon) { + bool was_running = q6v5->running; int ret; q6v5->running = false; - /* Don't perform SMP2P dance if remote isn't running */ - if (q6v5->rproc->state != RPROC_RUNNING || qcom_sysmon_shutdown_acked(sysmon)) + /* A watchdog/fatal IRQ clears running; logical crashes still need a stop. */ + if (!was_running || qcom_sysmon_shutdown_acked(sysmon)) return 0; qcom_smem_state_update_bits(q6v5->state, diff --git a/drivers/remoteproc/qcom_q6v5_adsp.c b/drivers/remoteproc/qcom_q6v5_adsp.c index b5c8d6d38c9cbc..c81e6c33c74790 100644 --- a/drivers/remoteproc/qcom_q6v5_adsp.c +++ b/drivers/remoteproc/qcom_q6v5_adsp.c @@ -355,6 +355,7 @@ static int adsp_map_carveout(struct rproc *rproc) return ret; sid = args.args[0] & SID_MASK_DEFAULT; + of_node_put(args.np); /* Add SID configuration for ADSP Firmware to SMMU */ iova = adsp->mem_phys | (sid << 32); diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c index ae78f5c7c1b69e..eb14308e7aef0a 100644 --- a/drivers/remoteproc/qcom_q6v5_mss.c +++ b/drivers/remoteproc/qcom_q6v5_mss.c @@ -162,6 +162,7 @@ struct rproc_hexagon_res { char **active_clk_names; char **proxy_pd_names; int version; + int ssctl_id; bool need_mem_protection; bool need_pas_mem_setup; bool has_alt_reset; @@ -2191,7 +2192,7 @@ static int q6v5_probe(struct platform_device *pdev) qcom_add_smd_subdev(rproc, &qproc->smd_subdev); qcom_add_pdm_subdev(rproc, &qproc->pdm_subdev); qcom_add_ssr_subdev(rproc, &qproc->ssr_subdev, "mpss"); - qproc->sysmon = qcom_add_sysmon_subdev(rproc, "modem", 0x12); + qproc->sysmon = qcom_add_sysmon_subdev(rproc, "modem", desc->ssctl_id); if (IS_ERR(qproc->sysmon)) { ret = PTR_ERR(qproc->sysmon); goto remove_subdevs; @@ -2271,6 +2272,7 @@ static const struct rproc_hexagon_res sc7180_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_SC7180, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res sc7280_mss = { @@ -2301,6 +2303,7 @@ static const struct rproc_hexagon_res sc7280_mss = { .has_ext_cntl_regs = true, .has_vq6 = true, .version = MSS_SC7280, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res sdm660_mss = { @@ -2334,6 +2337,7 @@ static const struct rproc_hexagon_res sdm660_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_SDM660, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res sdm845_mss = { @@ -2371,6 +2375,7 @@ static const struct rproc_hexagon_res sdm845_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_SDM845, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8998_mss = { @@ -2404,6 +2409,7 @@ static const struct rproc_hexagon_res msm8998_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8998, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8996_mss = { @@ -2444,6 +2450,7 @@ static const struct rproc_hexagon_res msm8996_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8996, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res mdm9607_mss = { @@ -2479,6 +2486,7 @@ static const struct rproc_hexagon_res mdm9607_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MDM9607, + .ssctl_id = 0x22, }; static const struct rproc_hexagon_res msm8909_mss = { @@ -2515,6 +2523,7 @@ static const struct rproc_hexagon_res msm8909_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8909, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8916_mss = { @@ -2562,6 +2571,7 @@ static const struct rproc_hexagon_res msm8916_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8916, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8917_mss = { @@ -2606,6 +2616,7 @@ static const struct rproc_hexagon_res msm8917_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8917, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8937_mss = { @@ -2650,6 +2661,7 @@ static const struct rproc_hexagon_res msm8937_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8937, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8940_mss = { @@ -2694,6 +2706,7 @@ static const struct rproc_hexagon_res msm8940_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8940, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8953_mss = { @@ -2731,6 +2744,7 @@ static const struct rproc_hexagon_res msm8953_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8953, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8974_mss = { @@ -2785,6 +2799,7 @@ static const struct rproc_hexagon_res msm8974_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8974, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8226_mss = { @@ -2824,6 +2839,7 @@ static const struct rproc_hexagon_res msm8226_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8226, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8926_mss = { @@ -2871,6 +2887,7 @@ static const struct rproc_hexagon_res msm8926_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8926, + .ssctl_id = 0x12, }; static const struct of_device_id q6v5_of_match[] = { diff --git a/drivers/remoteproc/qcom_q6v5_pas.c b/drivers/remoteproc/qcom_q6v5_pas.c index 7ab89695282258..cb3e51d6f7cecb 100644 --- a/drivers/remoteproc/qcom_q6v5_pas.c +++ b/drivers/remoteproc/qcom_q6v5_pas.c @@ -60,6 +60,7 @@ struct qcom_pas_data { int region_assign_count; bool region_assign_shared; int region_assign_vmid; + bool early_boot; }; struct qcom_pas { @@ -504,6 +505,67 @@ static unsigned long qcom_pas_panic(struct rproc *rproc) return qcom_q6v5_panic(&pas->q6v5); } +static int qcom_pas_attach(struct rproc *rproc) +{ + struct qcom_pas *pas = rproc->priv; + bool ready_state; + bool crash_state; + bool stop_state; + int ret; + + pas->q6v5.handover_issued = true; + enable_irq(pas->q6v5.handover_irq); + + pas->q6v5.running = true; + ret = irq_get_irqchip_state(pas->q6v5.fatal_irq, + IRQCHIP_STATE_LINE_LEVEL, &crash_state); + if (ret) + goto disable_running; + + if (crash_state) { + dev_err(pas->dev, "Subsystem has crashed before driver probe\n"); + rproc_report_crash(rproc, RPROC_FATAL_ERROR); + ret = -EINVAL; + goto disable_running; + } + + ret = irq_get_irqchip_state(pas->q6v5.stop_irq, + IRQCHIP_STATE_LINE_LEVEL, &stop_state); + if (ret) + goto disable_running; + + if (stop_state || qcom_sysmon_shutdown_irq_state(pas->sysmon)) { + dev_info(pas->dev, "Subsystem found stop state set. Falling back to start.\n"); + goto unroll_attach; + } + + ret = irq_get_irqchip_state(pas->q6v5.ready_irq, + IRQCHIP_STATE_LINE_LEVEL, &ready_state); + if (ret) + goto disable_running; + + if (unlikely(!ready_state)) { + /* + * The bootloader may not support early boot, mark the state as + * RPROC_OFFLINE so that the PAS driver can load the firmware and + * start the remoteproc. + */ + dev_err(pas->dev, "Failed to get subsystem ready interrupt\n"); + goto unroll_attach; + } + + return 0; + +unroll_attach: + pas->rproc->state = RPROC_OFFLINE; + ret = -EINVAL; +disable_running: + disable_irq(pas->q6v5.handover_irq); + pas->q6v5.running = false; + + return ret; +} + static const struct rproc_ops qcom_pas_ops = { .unprepare = qcom_pas_unprepare, .start = qcom_pas_start, @@ -512,6 +574,7 @@ static const struct rproc_ops qcom_pas_ops = { .parse_fw = qcom_pas_parse_firmware, .load = qcom_pas_load, .panic = qcom_pas_panic, + .attach = qcom_pas_attach, }; static const struct rproc_ops qcom_pas_minidump_ops = { @@ -523,6 +586,7 @@ static const struct rproc_ops qcom_pas_minidump_ops = { .load = qcom_pas_load, .panic = qcom_pas_panic, .coredump = qcom_pas_minidump, + .attach = qcom_pas_attach, }; static int qcom_pas_init_clock(struct qcom_pas *pas) @@ -849,6 +913,10 @@ static int qcom_pas_probe(struct platform_device *pdev) pas->pas_ctx->use_tzmem = rproc->has_iommu; pas->dtb_pas_ctx->use_tzmem = rproc->has_iommu; + + if (desc->early_boot) + pas->rproc->state = RPROC_DETACHED; + ret = rproc_add(rproc); if (ret) goto remove_ssr_sysmon; diff --git a/drivers/remoteproc/qcom_sysmon.c b/drivers/remoteproc/qcom_sysmon.c index 913e3b750a869c..a0830a48b1f40f 100644 --- a/drivers/remoteproc/qcom_sysmon.c +++ b/drivers/remoteproc/qcom_sysmon.c @@ -736,6 +736,25 @@ bool qcom_sysmon_shutdown_acked(struct qcom_sysmon *sysmon) } EXPORT_SYMBOL_GPL(qcom_sysmon_shutdown_acked); +bool qcom_sysmon_shutdown_irq_state(struct qcom_sysmon *sysmon) +{ + bool shutdown_state; + int ret; + + if (!sysmon) + return false; + + ret = irq_get_irqchip_state(sysmon->shutdown_irq, + IRQCHIP_STATE_LINE_LEVEL, &shutdown_state); + if (ret) { + dev_warn(sysmon->dev, "failed to get shutdown_state: %d\n", ret); + return false; + } + + return shutdown_state; +} +EXPORT_SYMBOL_GPL(qcom_sysmon_shutdown_irq_state); + /** * sysmon_probe() - probe sys_mon channel * @rpdev: rpmsg device handle diff --git a/drivers/remoteproc/qcom_wcnss.c b/drivers/remoteproc/qcom_wcnss.c index 4add9037dbd5a1..5e4a623b89251b 100644 --- a/drivers/remoteproc/qcom_wcnss.c +++ b/drivers/remoteproc/qcom_wcnss.c @@ -441,25 +441,31 @@ static void wcnss_release_pds(struct qcom_wcnss *wcnss) } static int wcnss_init_regulators(struct qcom_wcnss *wcnss, - const struct wcnss_vreg_info *info, - int num_vregs, int num_pd_vregs) + const struct wcnss_data *data) { + const struct wcnss_vreg_info *info = data->vregs; struct regulator_bulk_data *bulk; + size_t i, possible_pds = 0, num_vregs = data->num_vregs; int ret; - int i; + + for (i = 0; i < WCNSS_MAX_PDS; i++) + if (data->pd_names[i]) + possible_pds++; /* * If attaching the power domains suceeded we can skip requesting * the regulators for the power domains. For old device trees we need to * reserve extra space to manage them through the regulator interface. */ - if (wcnss->num_pds) { + if (possible_pds >= num_vregs) { + /* Do nothing if vregs do not include PD regulators (pronto-v3) */ + } else if (wcnss->num_pds) { info += wcnss->num_pds; /* Handle single power domain case */ - if (wcnss->num_pds < num_pd_vregs) - num_vregs += num_pd_vregs - wcnss->num_pds; + if (wcnss->num_pds < data->num_pd_vregs) + num_vregs += data->num_pd_vregs - wcnss->num_pds; } else { - num_vregs += num_pd_vregs; + num_vregs += data->num_pd_vregs; } bulk = devm_kcalloc(wcnss->dev, @@ -607,8 +613,7 @@ static int wcnss_probe(struct platform_device *pdev) if (ret && (ret != -ENODATA || !data->num_pd_vregs)) return ret; - ret = wcnss_init_regulators(wcnss, data->vregs, data->num_vregs, - data->num_pd_vregs); + ret = wcnss_init_regulators(wcnss, data); if (ret) goto detach_pds; diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c index f003be006b1bf8..527996e8ecdaaa 100644 --- a/drivers/remoteproc/remoteproc_core.c +++ b/drivers/remoteproc/remoteproc_core.c @@ -1668,18 +1668,21 @@ static void rproc_auto_boot_callback(const struct firmware *fw, void *context) release_firmware(fw); } +static void rproc_attach_work(struct work_struct *work) +{ + struct rproc *rproc = container_of(work, struct rproc, attach_work); + + rproc_boot(rproc); +} + static int rproc_trigger_auto_boot(struct rproc *rproc) { int ret; - /* - * Since the remote processor is in a detached state, it has already - * been booted by another entity. As such there is no point in waiting - * for a firmware image to be loaded, we can simply initiate the process - * of attaching to it immediately. - */ - if (rproc->state == RPROC_DETACHED) - return rproc_boot(rproc); + if (rproc->state == RPROC_DETACHED) { + schedule_work(&rproc->attach_work); + return 0; + } /* * We're initiating an asynchronous firmware loading, so we can @@ -1828,6 +1831,11 @@ int rproc_trigger_recovery(struct rproc *rproc) if (ret) return ret; + if (READ_ONCE(rproc->deleting)) { + ret = -ENODEV; + goto unlock_mutex; + } + /* State could have changed before we got the mutex */ if (rproc->state != RPROC_CRASHED) goto unlock_mutex; @@ -1860,6 +1868,11 @@ static void rproc_crash_handler_work(struct work_struct *work) mutex_lock(&rproc->lock); + if (READ_ONCE(rproc->deleting)) { + mutex_unlock(&rproc->lock); + goto out; + } + if (rproc->state == RPROC_CRASHED) { /* handle only the first crash detected */ mutex_unlock(&rproc->lock); @@ -1915,9 +1928,9 @@ int rproc_boot(struct rproc *rproc) return ret; } - if (rproc->state == RPROC_DELETED) { + if (READ_ONCE(rproc->deleting)) { ret = -ENODEV; - dev_err(dev, "can't boot deleted rproc %s\n", rproc->name); + dev_err(dev, "can't boot deleting rproc %s\n", rproc->name); goto unlock_mutex; } @@ -1979,6 +1992,7 @@ EXPORT_SYMBOL(rproc_boot); int rproc_shutdown(struct rproc *rproc) { struct device *dev = &rproc->dev; + bool crashed; int ret; ret = mutex_lock_interruptible(&rproc->lock); @@ -1988,16 +2002,18 @@ int rproc_shutdown(struct rproc *rproc) } if (rproc->state != RPROC_RUNNING && - rproc->state != RPROC_ATTACHED) { + rproc->state != RPROC_ATTACHED && + rproc->state != RPROC_CRASHED) { ret = -EINVAL; goto out; } + crashed = rproc->state == RPROC_CRASHED; /* if the remote proc is still needed, bail out */ if (!atomic_dec_and_test(&rproc->power)) goto out; - ret = rproc_stop(rproc, false); + ret = rproc_stop(rproc, crashed); if (ret) { atomic_inc(&rproc->power); goto out; @@ -2506,7 +2522,9 @@ struct rproc *rproc_alloc(struct device *dev, const char *name, INIT_LIST_HEAD(&rproc->subdevs); INIT_LIST_HEAD(&rproc->dump_segments); + INIT_WORK(&rproc->attach_work, rproc_attach_work); INIT_WORK(&rproc->crash_handler, rproc_crash_handler_work); + spin_lock_init(&rproc->crash_handler_lock); rproc->state = RPROC_OFFLINE; @@ -2570,16 +2588,21 @@ EXPORT_SYMBOL(rproc_put); */ int rproc_del(struct rproc *rproc) { + unsigned long flags; + if (!rproc) return -EINVAL; + spin_lock_irqsave(&rproc->crash_handler_lock, flags); + WRITE_ONCE(rproc->deleting, true); + spin_unlock_irqrestore(&rproc->crash_handler_lock, flags); + + if (cancel_work_sync(&rproc->crash_handler)) + pm_relax(rproc->dev.parent); + /* TODO: make sure this works with rproc->power > 1 */ rproc_shutdown(rproc); - mutex_lock(&rproc->lock); - rproc->state = RPROC_DELETED; - mutex_unlock(&rproc->lock); - rproc_delete_debug_dir(rproc); /* the rproc is downref'ed as soon as it's removed from the klist */ @@ -2691,18 +2714,26 @@ EXPORT_SYMBOL(rproc_get_by_child); */ void rproc_report_crash(struct rproc *rproc, enum rproc_crash_type type) { + unsigned long flags; + if (!rproc) { pr_err("NULL rproc pointer\n"); return; } + spin_lock_irqsave(&rproc->crash_handler_lock, flags); + if (READ_ONCE(rproc->deleting)) { + spin_unlock_irqrestore(&rproc->crash_handler_lock, flags); + return; + } + /* Prevent suspend while the remoteproc is being recovered */ pm_stay_awake(rproc->dev.parent); + queue_work(rproc_recovery_wq, &rproc->crash_handler); + spin_unlock_irqrestore(&rproc->crash_handler_lock, flags); dev_err(&rproc->dev, "crash detected in %s: type %s\n", rproc->name, rproc_crash_to_string(type)); - - queue_work(rproc_recovery_wq, &rproc->crash_handler); } EXPORT_SYMBOL(rproc_report_crash); diff --git a/drivers/remoteproc/remoteproc_sysfs.c b/drivers/remoteproc/remoteproc_sysfs.c index 138e752c5e4e08..925b0cdbe57788 100644 --- a/drivers/remoteproc/remoteproc_sysfs.c +++ b/drivers/remoteproc/remoteproc_sysfs.c @@ -168,7 +168,6 @@ static const char * const rproc_state_string[] = { [RPROC_SUSPENDED] = "suspended", [RPROC_RUNNING] = "running", [RPROC_CRASHED] = "crashed", - [RPROC_DELETED] = "deleted", [RPROC_ATTACHED] = "attached", [RPROC_DETACHED] = "detached", [RPROC_LAST] = "invalid", diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 2f09f4b78bd3bc..dd422c56fbb147 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -1196,8 +1196,7 @@ static u64 mpam_msmon_overflow_val(enum mpam_device_features type, { u64 overflow_val = __mpam_msmon_overflow_val(type); - if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc) && - type != mpam_feat_msmon_mbwu_63counter) + if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc)) overflow_val *= 64; return overflow_val; @@ -1293,8 +1292,7 @@ static void __ris_msmon_read(void *arg) now = FIELD_GET(MSMON___VALUE, now); } - if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc) && - m->type != mpam_feat_msmon_mbwu_63counter) + if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc)) now *= 64; if (nrdy) @@ -2025,6 +2023,9 @@ static void mpam_msc_drv_remove(struct platform_device *pdev) { struct mpam_msc *msc = platform_get_drvdata(pdev); + if (!msc) + return; + mutex_lock(&mpam_list_lock); mpam_msc_destroy(msc); mutex_unlock(&mpam_list_lock); @@ -2138,6 +2139,7 @@ static int mpam_msc_drv_probe(struct platform_device *pdev) static struct platform_driver mpam_msc_driver = { .driver = { .name = "mpam_msc", + .suppress_bind_attrs = true, }, .probe = mpam_msc_drv_probe, .remove = mpam_msc_drv_remove, diff --git a/drivers/rpmsg/qcom_glink_native.c b/drivers/rpmsg/qcom_glink_native.c index d9d4468e4cbdfd..fda1ddda05016c 100644 --- a/drivers/rpmsg/qcom_glink_native.c +++ b/drivers/rpmsg/qcom_glink_native.c @@ -1418,9 +1418,6 @@ static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept) channel->ept.cb = NULL; spin_unlock_irqrestore(&channel->recv_lock, flags); - /* Decouple the potential rpdev from the channel */ - qcom_glink_remove_rpmsg_device(glink, channel); - qcom_glink_send_close_req(glink, channel); } diff --git a/drivers/rtc/rtc-gamecube.c b/drivers/rtc/rtc-gamecube.c index 045d5d45ab4b07..889028cecf4a56 100644 --- a/drivers/rtc/rtc-gamecube.c +++ b/drivers/rtc/rtc-gamecube.c @@ -355,9 +355,7 @@ static int gamecube_rtc_probe(struct platform_device *pdev) rtc->range_max = U32_MAX; rtc->ops = &gamecube_rtc_ops; - devm_rtc_register_device(rtc); - - return 0; + return devm_rtc_register_device(rtc); } static const struct of_device_id gamecube_rtc_of_match[] = { diff --git a/drivers/rtc/rtc-pcf85363.c b/drivers/rtc/rtc-pcf85363.c index 540042b9eec8f5..ccc7834e575997 100644 --- a/drivers/rtc/rtc-pcf85363.c +++ b/drivers/rtc/rtc-pcf85363.c @@ -426,8 +426,8 @@ static int pcf85363_probe(struct i2c_client *client) err = pcf85363_load_capacitance(pcf85363, client->dev.of_node); if (err < 0) - dev_warn(&client->dev, "failed to set xtal load capacitance: %d", - err); + return dev_err_probe(&client->dev, err, + "failed to set xtal load capacitance\n"); pcf85363->rtc->ops = &rtc_ops; pcf85363->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000; @@ -436,9 +436,16 @@ static int pcf85363_probe(struct i2c_client *client) wakeup_source = device_property_read_bool(&client->dev, "wakeup-source"); if (client->irq > 0 || wakeup_source) { - regmap_write(pcf85363->regmap, CTRL_FLAGS, 0); - regmap_update_bits(pcf85363->regmap, CTRL_PIN_IO, - PIN_IO_INTAPM, PIN_IO_INTA_OUT); + err = regmap_write(pcf85363->regmap, CTRL_FLAGS, 0); + if (err) + return dev_err_probe(&client->dev, err, + "failed to clear flags\n"); + + err = regmap_update_bits(pcf85363->regmap, CTRL_PIN_IO, + PIN_IO_INTAPM, PIN_IO_INTA_OUT); + if (err) + return dev_err_probe(&client->dev, err, + "failed to set interrupt pin mode\n"); } if (client->irq > 0) { diff --git a/drivers/rtc/rtc-pcf8563.c b/drivers/rtc/rtc-pcf8563.c index 81d13733b1e937..a90dc940474bf3 100644 --- a/drivers/rtc/rtc-pcf8563.c +++ b/drivers/rtc/rtc-pcf8563.c @@ -449,7 +449,9 @@ static struct clk *pcf8563_clkout_register_clk(struct pcf8563 *pcf8563) clk = devm_clk_register(&pcf8563->rtc->dev, &pcf8563->clkout_hw); if (!IS_ERR(clk)) - of_clk_add_provider(node, of_clk_src_simple_get, clk); + devm_of_clk_add_hw_provider(pcf8563->rtc->dev.parent, + of_clk_hw_simple_get, + &pcf8563->clkout_hw); return clk; } diff --git a/drivers/rtc/rtc-spacemit-p1.c b/drivers/rtc/rtc-spacemit-p1.c index 43ab62494bb4b3..1de7bd995d296c 100644 --- a/drivers/rtc/rtc-spacemit-p1.c +++ b/drivers/rtc/rtc-spacemit-p1.c @@ -57,8 +57,9 @@ static int p1_rtc_read_time(struct device *dev, struct rtc_time *t) u8 time[6]; int ret; - if (!regmap_test_bits(regmap, RTC_CTRL, RTC_EN)) - return -EINVAL; /* RTC is disabled */ + ret = regmap_test_bits(regmap, RTC_CTRL, RTC_EN); + if (ret <= 0) + return ret ?: -EINVAL; /* RTC is disabled or error */ ret = regmap_bulk_read(regmap, RTC_TIME, time, sizeof(time)); if (ret) diff --git a/drivers/rtc/rtc-zynqmp.c b/drivers/rtc/rtc-zynqmp.c index 2ae54804b87a47..5bcb7536e9734c 100644 --- a/drivers/rtc/rtc-zynqmp.c +++ b/drivers/rtc/rtc-zynqmp.c @@ -334,10 +334,9 @@ static int xlnx_rtc_probe(struct platform_device *pdev) /* Getting the rtc info */ xrtcdev->rtc_clk = devm_clk_get_optional(&pdev->dev, "rtc"); - if (IS_ERR(xrtcdev->rtc_clk)) { - if (PTR_ERR(xrtcdev->rtc_clk) != -EPROBE_DEFER) - dev_warn(&pdev->dev, "Device clock not found.\n"); - } + if (IS_ERR(xrtcdev->rtc_clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(xrtcdev->rtc_clk), + "Failed to get rtc clock\n"); xrtcdev->freq = clk_get_rate(xrtcdev->rtc_clk); if (!xrtcdev->freq) { ret = of_property_read_u32(pdev->dev.of_node, "calibration", diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c index 79052f2accbd34..11bcb8440e1c87 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_base.c +++ b/drivers/scsi/mpt3sas/mpt3sas_base.c @@ -5870,6 +5870,8 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc) if (ioc->pcie_sgl_dma_pool) { for (i = 0; i < ioc->scsiio_depth; i++) { + if (!ioc->pcie_sg_lookup[i].pcie_sgl) + continue; dma_pool_free(ioc->pcie_sgl_dma_pool, ioc->pcie_sg_lookup[i].pcie_sgl, ioc->pcie_sg_lookup[i].pcie_sgl_dma); diff --git a/drivers/scsi/qla2xxx/qla_attr.c b/drivers/scsi/qla2xxx/qla_attr.c index 14c3efd0e21093..b3b7cdf0cfa90e 100644 --- a/drivers/scsi/qla2xxx/qla_attr.c +++ b/drivers/scsi/qla2xxx/qla_attr.c @@ -580,7 +580,6 @@ qla2x00_sysfs_read_vpd(struct file *filp, struct kobject *kobj, ha->isp_ops->read_optrom(vha, ha->vpd, faddr, ha->vpd_size); mutex_unlock(&ha->optrom_mutex); - ha->isp_ops->read_optrom(vha, ha->vpd, faddr, ha->vpd_size); skip: return memory_read_from_buffer(buf, count, &off, ha->vpd, ha->vpd_size); } diff --git a/drivers/scsi/qla2xxx/qla_nx2.c b/drivers/scsi/qla2xxx/qla_nx2.c index 41ff6fbdb93332..04d7ab6ad037a3 100644 --- a/drivers/scsi/qla2xxx/qla_nx2.c +++ b/drivers/scsi/qla2xxx/qla_nx2.c @@ -3507,7 +3507,7 @@ qla8044_poll_flash_status_reg(struct scsi_qla_host *vha) msleep(QLA8044_FLASH_STATUS_REG_POLL_DELAY); } - if (!retries) + if (retries == -1) ret_val = QLA_FUNCTION_FAILED; return ret_val; diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index 599e75f3333438..a1b21ea14e5491 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -958,6 +958,21 @@ static unsigned char sd_setup_protect_cmnd(struct scsi_cmnd *scmd, return protect; } +static void sd_uninit_command(struct scsi_cmnd *cmd) +{ + struct request *rq = scsi_cmd_to_rq(cmd); + struct scsi_device *sdp = cmd->device; + + if (!(rq->rq_flags & RQF_SPECIAL_PAYLOAD)) + return; + + if (sdp->sector_size > PAGE_SIZE) + mempool_free(rq->special_vec.bv_page, sd_large_page_pool); + else + mempool_free(rq->special_vec.bv_page, sd_page_pool); + rq->rq_flags &= ~RQF_SPECIAL_PAYLOAD; +} + static void *sd_set_special_bvec(struct scsi_cmnd *cmd, unsigned int data_len) { struct page *page; @@ -990,6 +1005,7 @@ static blk_status_t sd_setup_unmap_cmnd(struct scsi_cmnd *cmd) u32 nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq)); unsigned int data_len = 24; char *buf; + blk_status_t ret; buf = sd_set_special_bvec(cmd, data_len); if (!buf) @@ -1008,7 +1024,10 @@ static blk_status_t sd_setup_unmap_cmnd(struct scsi_cmnd *cmd) cmd->transfersize = data_len; rq->timeout = SD_TIMEOUT; - return scsi_alloc_sgtables(cmd); + ret = scsi_alloc_sgtables(cmd); + if (ret != BLK_STS_OK) + sd_uninit_command(cmd); + return ret; } static void sd_config_atomic(struct scsi_disk *sdkp, struct queue_limits *lim) @@ -1079,6 +1098,7 @@ static blk_status_t sd_setup_write_same16_cmnd(struct scsi_cmnd *cmd, u64 lba = sectors_to_logical(sdp, blk_rq_pos(rq)); u32 nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq)); u32 data_len = sdp->sector_size; + blk_status_t ret; if (!sd_set_special_bvec(cmd, data_len)) return BLK_STS_RESOURCE; @@ -1094,7 +1114,10 @@ static blk_status_t sd_setup_write_same16_cmnd(struct scsi_cmnd *cmd, cmd->transfersize = data_len; rq->timeout = unmap ? SD_TIMEOUT : SD_WRITE_SAME_TIMEOUT; - return scsi_alloc_sgtables(cmd); + ret = scsi_alloc_sgtables(cmd); + if (ret != BLK_STS_OK) + sd_uninit_command(cmd); + return ret; } static blk_status_t sd_setup_write_same10_cmnd(struct scsi_cmnd *cmd, @@ -1106,6 +1129,7 @@ static blk_status_t sd_setup_write_same10_cmnd(struct scsi_cmnd *cmd, u64 lba = sectors_to_logical(sdp, blk_rq_pos(rq)); u32 nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq)); u32 data_len = sdp->sector_size; + blk_status_t ret; if (!sd_set_special_bvec(cmd, data_len)) return BLK_STS_RESOURCE; @@ -1121,7 +1145,10 @@ static blk_status_t sd_setup_write_same10_cmnd(struct scsi_cmnd *cmd, cmd->transfersize = data_len; rq->timeout = unmap ? SD_TIMEOUT : SD_WRITE_SAME_TIMEOUT; - return scsi_alloc_sgtables(cmd); + ret = scsi_alloc_sgtables(cmd); + if (ret != BLK_STS_OK) + sd_uninit_command(cmd); + return ret; } static blk_status_t sd_setup_write_zeroes_cmnd(struct scsi_cmnd *cmd) @@ -1550,20 +1577,6 @@ static blk_status_t sd_init_command(struct scsi_cmnd *cmd) } } -static void sd_uninit_command(struct scsi_cmnd *SCpnt) -{ - struct request *rq = scsi_cmd_to_rq(SCpnt); - struct scsi_device *sdp = SCpnt->device; - unsigned sector_size = sdp->sector_size; - - if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) { - if (sector_size > PAGE_SIZE) - mempool_free(rq->special_vec.bv_page, sd_large_page_pool); - else - mempool_free(rq->special_vec.bv_page, sd_page_pool); - } -} - static bool sd_need_revalidate(struct gendisk *disk, struct scsi_disk *sdkp) { if (sdkp->device->removable || sdkp->write_prot) { @@ -2406,8 +2419,8 @@ static int sd_done(struct scsi_cmnd *SCpnt) } sdkp->medium_access_timed_out = 0; - if (!scsi_status_is_check_condition(result) && - (!sense_valid || sense_deferred)) + if (!scsi_status_is_check_condition(result) || + !sense_valid || sense_deferred) goto out; switch (sshdr.sense_key) { @@ -4089,7 +4102,9 @@ static int sd_probe(struct scsi_device *sdp) if (sdp->sector_size > PAGE_SIZE) { if (sd_large_pool_create()) { error = -ENOMEM; - goto out_free_index; + device_unregister(&sdkp->disk_dev); + put_disk(gd); + goto out; } } diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c index 5ec583dc2e7df5..3a75b9fbedf45a 100644 --- a/drivers/scsi/smartpqi/smartpqi_init.c +++ b/drivers/scsi/smartpqi/smartpqi_init.c @@ -66,6 +66,12 @@ static struct pqi_cmd_priv *pqi_cmd_priv(struct scsi_cmnd *cmd) return scsi_cmd_priv(cmd); } +static int pqi_init_cmd_priv(struct Scsi_Host *shost, struct scsi_cmnd *cmd) +{ + memset(pqi_cmd_priv(cmd), 0, sizeof(struct pqi_cmd_priv)); + return 0; +} + static void pqi_verify_structures(void); static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info, enum pqi_ctrl_shutdown_reason ctrl_shutdown_reason); @@ -5958,6 +5964,17 @@ void pqi_prep_for_scsi_done(struct scsi_cmnd *scmd) struct pqi_scsi_dev *device; struct completion *wait; + /* + * Clear the AIO-retry marker on final completion so the tag + * starts clean on its next dispatch. On DID_IMM_RETRY leave + * it intact: pqi_aio_io_complete() sets DID_IMM_RETRY and + * bumps the marker to steer the requeue onto the RAID path, + * and pqi_process_raid_io_error() consumes the non-zero + * marker to offline a misbehaving drive. + */ + if (host_byte(scmd->result) != DID_IMM_RETRY) + pqi_cmd_priv(scmd)->this_residual = 0; + if (!scmd->device) { set_host_byte(scmd, DID_NO_CONNECT); return; @@ -7612,6 +7629,7 @@ static const struct scsi_host_template pqi_driver_template = { .sdev_groups = pqi_sdev_groups, .shost_groups = pqi_shost_groups, .cmd_size = sizeof(struct pqi_cmd_priv), + .init_cmd_priv = pqi_init_cmd_priv, }; static int pqi_register_scsi(struct pqi_ctrl_info *ctrl_info) diff --git a/drivers/soc/fsl/qe/gpio.c b/drivers/soc/fsl/qe/gpio.c index 66828f2a35774e..6d8f4d549fe290 100644 --- a/drivers/soc/fsl/qe/gpio.c +++ b/drivers/soc/fsl/qe/gpio.c @@ -135,6 +135,30 @@ static int qe_gpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) return 0; } +static int qe_gpio_get_direction(struct gpio_chip *gc, unsigned int gpio) +{ + struct qe_gpio_chip *qe_gc = gpiochip_get_data(gc); + struct qe_pio_regs __iomem *regs = qe_gc->regs; + unsigned long flags; + u32 val, mask; + + spin_lock_irqsave(&qe_gc->lock, flags); + + if (gpio < QE_PIO_PINS / 2) + val = ioread32be(®s->cpdir1); + else + val = ioread32be(®s->cpdir2); + + spin_unlock_irqrestore(&qe_gc->lock, flags); + + mask = (u32)QE_PIO_DIR_OUT << (QE_PIO_PINS - 2 - (gpio % (QE_PIO_PINS / 2)) * 2); + + if (val & mask) + return GPIO_LINE_DIRECTION_OUT; + else + return GPIO_LINE_DIRECTION_IN; +} + struct qe_pin { /* * The qe_gpio_chip name is unfortunate, we should change that to @@ -308,6 +332,7 @@ static int qe_gpio_probe(struct platform_device *ofdev) gc->ngpio = QE_PIO_PINS; gc->direction_input = qe_gpio_dir_in; gc->direction_output = qe_gpio_dir_out; + gc->get_direction = qe_gpio_get_direction; gc->get = qe_gpio_get; gc->set = qe_gpio_set; gc->set_multiple = qe_gpio_set_multiple; diff --git a/drivers/soc/fsl/qe/qe_ic.c b/drivers/soc/fsl/qe/qe_ic.c index 943911053af68f..bccc7d612009b3 100644 --- a/drivers/soc/fsl/qe/qe_ic.c +++ b/drivers/soc/fsl/qe/qe_ic.c @@ -473,7 +473,6 @@ static struct platform_driver qe_ic_driver = static int __init qe_ic_of_init(void) { - platform_driver_register(&qe_ic_driver); - return 0; + return platform_driver_register(&qe_ic_driver); } subsys_initcall(qe_ic_of_init); diff --git a/drivers/soc/fsl/qe/qe_io.c b/drivers/soc/fsl/qe/qe_io.c index a5e2d0e5ab511f..150913fce98189 100644 --- a/drivers/soc/fsl/qe/qe_io.c +++ b/drivers/soc/fsl/qe/qe_io.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -184,3 +185,17 @@ int par_io_of_config(struct device_node *np) return 0; } EXPORT_SYMBOL(par_io_of_config); + +static int __init par_io_populate(void) +{ + struct device_node *np = of_find_node_by_type(NULL, "par_io"); + + if (!np) + return 0; + + of_platform_default_populate(np, NULL, NULL); + of_node_put(np); + + return 0; +} +arch_initcall(par_io_populate); diff --git a/drivers/soc/qcom/pmic_glink_altmode.c b/drivers/soc/qcom/pmic_glink_altmode.c index 619bad2c27eeb3..13c434f8d03a64 100644 --- a/drivers/soc/qcom/pmic_glink_altmode.c +++ b/drivers/soc/qcom/pmic_glink_altmode.c @@ -53,7 +53,7 @@ struct usbc_sc8280x_tbt_data { /* This field is NOP on USB4, all cables support rounded rates by spec */ u8 rounded_cable : 1; u8 power_limited : 1; - u8 res[11]; + u8 res[7]; }; struct usbc_notify { @@ -74,6 +74,7 @@ struct usbc_notify { struct usbc_sc8280x_dp_data dp; struct usbc_sc8280x_tbt_data tbt; } extended_data; + u32 reserved; }; struct usbc_sc8180x_notify { diff --git a/drivers/soc/qcom/rpmh-rsc.c b/drivers/soc/qcom/rpmh-rsc.c index c6f7d5c9c493d9..66928ca40b9aad 100644 --- a/drivers/soc/qcom/rpmh-rsc.c +++ b/drivers/soc/qcom/rpmh-rsc.c @@ -944,17 +944,30 @@ static int rpmh_rsc_pd_callback(struct notifier_block *nfb, return NOTIFY_OK; } +static void rpmh_rsc_pd_detach(void *data) +{ + dev_pm_genpd_remove_notifier(data); +} + static int rpmh_rsc_pd_attach(struct rsc_drv *drv, struct device *dev) { int ret; - pm_runtime_enable(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; + drv->genpd_nb.notifier_call = rpmh_rsc_pd_callback; ret = dev_pm_genpd_add_notifier(dev, &drv->genpd_nb); if (ret) - pm_runtime_disable(dev); + return ret; - return ret; + return devm_add_action_or_reset(dev, rpmh_rsc_pd_detach, dev); +} + +static void rpmh_rsc_cpu_pm_unregister(void *data) +{ + cpu_pm_unregister_notifier(data); } static int rpmh_probe_tcs_config(struct platform_device *pdev, struct rsc_drv *drv) @@ -1107,7 +1120,15 @@ static int rpmh_rsc_probe(struct platform_device *pdev) return ret; } else { drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback; - cpu_pm_register_notifier(&drv->rsc_pm); + ret = cpu_pm_register_notifier(&drv->rsc_pm); + if (ret) + return ret; + + ret = devm_add_action_or_reset(&pdev->dev, + rpmh_rsc_cpu_pm_unregister, + &drv->rsc_pm); + if (ret) + return ret; } } @@ -1122,13 +1143,7 @@ static int rpmh_rsc_probe(struct platform_device *pdev) dev_set_drvdata(&pdev->dev, drv); drv->dev = &pdev->dev; - ret = devm_of_platform_populate(&pdev->dev); - if (ret && pdev->dev.pm_domain) { - dev_pm_genpd_remove_notifier(&pdev->dev); - pm_runtime_disable(&pdev->dev); - } - - return ret; + return devm_of_platform_populate(&pdev->dev); } static const struct of_device_id rpmh_drv_match[] = { diff --git a/drivers/soc/renesas/Kconfig b/drivers/soc/renesas/Kconfig index 2ab150d04bb1f1..fdf18ed2dfc218 100644 --- a/drivers/soc/renesas/Kconfig +++ b/drivers/soc/renesas/Kconfig @@ -356,7 +356,6 @@ config ARCH_R8A779H0 config ARCH_R8A78000 bool "ARM64 Platform support for R8A78000 (R-Car X5H)" default y if ARCH_RENESAS - default ARCH_RENESAS select ARCH_RCAR_GEN5 help This enables support for the Renesas R-Car X5H SoC. diff --git a/drivers/soc/samsung/exynos-pmu.c b/drivers/soc/samsung/exynos-pmu.c index d58376c38179bd..f5fcdde9750e27 100644 --- a/drivers/soc/samsung/exynos-pmu.c +++ b/drivers/soc/samsung/exynos-pmu.c @@ -167,8 +167,8 @@ static const struct mfd_cell exynos_pmu_devs[] = { */ struct regmap *exynos_get_pmu_regmap(void) { - struct device_node *np = of_find_matching_node(NULL, - exynos_pmu_of_device_ids); + struct device_node *np __free(device_node) = + of_find_matching_node(NULL, exynos_pmu_of_device_ids); if (np) return exynos_get_pmu_regmap_by_phandle(np, NULL); return ERR_PTR(-ENODEV); diff --git a/drivers/soc/ti/knav_qmss.h b/drivers/soc/ti/knav_qmss.h index 037dc1b3664535..8a624fbda84ab1 100644 --- a/drivers/soc/ti/knav_qmss.h +++ b/drivers/soc/ti/knav_qmss.h @@ -304,6 +304,7 @@ struct knav_device { struct list_head pools; struct list_head pdsps; struct list_head qmgrs; + struct dentry *debugfs_file; enum qmss_version version; }; diff --git a/drivers/soc/ti/knav_qmss_queue.c b/drivers/soc/ti/knav_qmss_queue.c index 7410b63af0e627..3e4041454f6943 100644 --- a/drivers/soc/ti/knav_qmss_queue.c +++ b/drivers/soc/ti/knav_qmss_queue.c @@ -1849,8 +1849,9 @@ static int knav_queue_probe(struct platform_device *pdev) goto err; } - debugfs_create_file("qmss", S_IFREG | S_IRUGO, NULL, NULL, - &knav_queue_debug_fops); + knav_qdev->debugfs_file = + debugfs_create_file("qmss", 0444, NULL, NULL, + &knav_queue_debug_fops); device_ready = true; return 0; @@ -1868,6 +1869,8 @@ static void knav_queue_remove(struct platform_device *pdev) struct knav_device *kdev = platform_get_drvdata(pdev); device_ready = false; + debugfs_remove(kdev->debugfs_file); + kdev->debugfs_file = NULL; knav_queue_stop_pdsps(kdev); knav_queue_free_regions(kdev); knav_free_queue_ranges(kdev); diff --git a/drivers/soundwire/cadence_master.c b/drivers/soundwire/cadence_master.c index eb66345a6a42ba..98ed337fb9b016 100644 --- a/drivers/soundwire/cadence_master.c +++ b/drivers/soundwire/cadence_master.c @@ -2358,7 +2358,9 @@ int sdw_cdns_prepare_write_dma_buffer(u8 dev_num, struct sdw_bpt_section *sec, i p_data = sec[i].buf; while (section_size >= data_per_frame) { - header[1] = data_per_frame; + header[0] &= ~BIT(0); + header[0] |= (data_per_frame >> 8) & BIT(0); + header[1] = data_per_frame & 0xFF; header[2] = start_register >> 24 & 0xFF; header[3] = start_register >> 16 & 0xFF; header[4] = start_register >> 8 & 0xFF; @@ -2384,7 +2386,9 @@ int sdw_cdns_prepare_write_dma_buffer(u8 dev_num, struct sdw_bpt_section *sec, i } if (section_size) { - header[1] = section_size; + header[0] &= ~BIT(0); + header[0] |= (section_size >> 8) & BIT(0); + header[1] = section_size & 0xFF; header[2] = start_register >> 24 & 0xFF; header[3] = start_register >> 16 & 0xFF; header[4] = start_register >> 8 & 0xFF; @@ -2435,7 +2439,9 @@ int sdw_cdns_prepare_read_dma_buffer(u8 dev_num, struct sdw_bpt_section *sec, in start_register = sec[i].addr; data_size = sec[i].len; while (data_size >= data_per_frame) { - header[1] = data_per_frame; + header[0] &= ~BIT(0); + header[0] |= (data_per_frame >> 8) & BIT(0); + header[1] = data_per_frame & 0xFF; header[2] = start_register >> 24 & 0xFF; header[3] = start_register >> 16 & 0xFF; header[4] = start_register >> 8 & 0xFF; @@ -2459,7 +2465,9 @@ int sdw_cdns_prepare_read_dma_buffer(u8 dev_num, struct sdw_bpt_section *sec, in } if (data_size) { - header[1] = data_size; + header[0] &= ~BIT(0); + header[0] |= (data_size >> 8) & BIT(0); + header[1] = data_size & 0xFF; header[2] = start_register >> 24 & 0xFF; header[3] = start_register >> 16 & 0xFF; header[4] = start_register >> 8 & 0xFF; @@ -2482,7 +2490,9 @@ int sdw_cdns_prepare_read_dma_buffer(u8 dev_num, struct sdw_bpt_section *sec, in /* Add fake frame */ header[0] &= ~GENMASK(7, 6); /* Set inactive flag in BPT/BRA frame heade */ while (fake_size >= data_per_frame) { - header[1] = data_per_frame; + header[0] &= ~BIT(0); + header[0] |= (data_per_frame >> 8) & BIT(0); + header[1] = data_per_frame & 0xFF; ret = sdw_cdns_prepare_read_pd0_buffer(header, SDW_CDNS_BRA_HDR, p_dma_buffer, dma_buffer_size, &dma_data_written, counter); @@ -2498,7 +2508,9 @@ int sdw_cdns_prepare_read_dma_buffer(u8 dev_num, struct sdw_bpt_section *sec, in } if (fake_size) { - header[1] = fake_size; + header[0] &= ~BIT(0); + header[0] |= (fake_size >> 8) & BIT(0); + header[1] = fake_size & 0xFF; ret = sdw_cdns_prepare_read_pd0_buffer(header, SDW_CDNS_BRA_HDR, p_dma_buffer, dma_buffer_size, &dma_data_written, counter); diff --git a/drivers/soundwire/qcom.c b/drivers/soundwire/qcom.c index 3d8f5a81eff195..b288218f64b420 100644 --- a/drivers/soundwire/qcom.c +++ b/drivers/soundwire/qcom.c @@ -1271,7 +1271,7 @@ static int qcom_swrm_stream_alloc_ports(struct qcom_swrm_ctrl *ctrl, else pn = find_first_zero_bit(port_mask, maxport); - if (pn > maxport) { + if (pn >= maxport) { dev_err(ctrl->dev, "All ports busy\n"); return -EBUSY; } diff --git a/drivers/spi/spi-amlogic-spisg.c b/drivers/spi/spi-amlogic-spisg.c index afc8af04638d3b..9049a87e9d0f2d 100644 --- a/drivers/spi/spi-amlogic-spisg.c +++ b/drivers/spi/spi-amlogic-spisg.c @@ -636,7 +636,7 @@ static int aml_spisg_target_abort(struct spi_controller *ctlr) static int aml_spisg_clk_init(struct spisg_device *spisg, void __iomem *base) { struct device *dev = &spisg->pdev->dev; - struct clk_init_data init; + struct clk_init_data init = {}; struct clk_divider *div; struct clk_div_table *tbl; char name[32]; diff --git a/drivers/spi/spi-davinci.c b/drivers/spi/spi-davinci.c index da7f2ae3a570a7..087735ed9c80c5 100644 --- a/drivers/spi/spi-davinci.c +++ b/drivers/spi/spi-davinci.c @@ -928,7 +928,7 @@ static int davinci_spi_probe(struct platform_device *pdev) int ret = 0; u32 spipc0; - host = spi_alloc_host(&pdev->dev, sizeof(struct davinci_spi)); + host = devm_spi_alloc_host(&pdev->dev, sizeof(struct davinci_spi)); if (host == NULL) { ret = -ENOMEM; goto err; @@ -1057,7 +1057,6 @@ free_dma: dma_release_channel(dspi->dma_tx); } free_host: - spi_controller_put(host); err: return ret; } @@ -1081,6 +1080,8 @@ static void davinci_spi_remove(struct platform_device *pdev) spi_bitbang_stop(&dspi->bitbang); + devm_free_irq(&pdev->dev, dspi->irq, dspi); + /* This bit needs to be cleared to disable dpsi->clk */ clear_io_bits(dspi->base + SPIGCR1, SPIGCR1_POWERDOWN_MASK); @@ -1088,8 +1089,6 @@ static void davinci_spi_remove(struct platform_device *pdev) dma_release_channel(dspi->dma_rx); dma_release_channel(dspi->dma_tx); } - - spi_controller_put(host); } static struct platform_driver davinci_spi_driver = { diff --git a/drivers/spi/spi-geni-qcom.c b/drivers/spi/spi-geni-qcom.c index 26e723cfea61ef..1fd9cf6e30f7ac 100644 --- a/drivers/spi/spi-geni-qcom.c +++ b/drivers/spi/spi-geni-qcom.c @@ -613,25 +613,30 @@ static int spi_geni_init(struct spi_geni_master *mas) u32 spi_tx_cfg, fifo_disable; int ret = -ENXIO; - pm_runtime_get_sync(mas->dev); + PM_RUNTIME_ACQUIRE_IF_ENABLED(mas->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret < 0) { + dev_err(mas->dev, "Failed to resume and get %d\n", ret); + return ret; + } proto = geni_se_read_proto(se); if (spi->target) { if (proto != GENI_SE_SPI_SLAVE) { dev_err(mas->dev, "Invalid proto %d\n", proto); - goto out_pm; + return -EINVAL; } spi_slv_setup(mas); } else if (proto == GENI_SE_INVALID_PROTO) { ret = geni_load_se_firmware(se, GENI_SE_SPI); if (ret) { dev_err(mas->dev, "spi master firmware load failed ret: %d\n", ret); - goto out_pm; + return ret; } } else if (proto != GENI_SE_SPI) { dev_err(mas->dev, "Invalid proto %d\n", proto); - goto out_pm; + return -EINVAL; } mas->tx_fifo_depth = geni_se_get_tx_fifo_depth(se); @@ -664,7 +669,7 @@ static int spi_geni_init(struct spi_geni_master *mas) dev_dbg(mas->dev, "Using GPI DMA mode for SPI\n"); break; } else if (ret == -EPROBE_DEFER) { - goto out_pm; + return ret; } /* * in case of failure to get gpi dma channel, we can still do the @@ -693,8 +698,6 @@ static int spi_geni_init(struct spi_geni_master *mas) writel(spi_tx_cfg, se->base + SE_SPI_TRANS_CFG); } -out_pm: - pm_runtime_put(mas->dev); return ret; } diff --git a/drivers/spi/spi-img-spfi.c b/drivers/spi/spi-img-spfi.c index aec724e3f8246a..a9310a1b0354f3 100644 --- a/drivers/spi/spi-img-spfi.c +++ b/drivers/spi/spi-img-spfi.c @@ -611,7 +611,7 @@ static int img_spfi_probe(struct platform_device *pdev) ret = PTR_ERR(spfi->tx_ch); spfi->tx_ch = NULL; if (ret == -EPROBE_DEFER) - goto disable_pm; + goto free_dma; } spfi->rx_ch = dma_request_chan(spfi->dev, "rx"); @@ -619,7 +619,7 @@ static int img_spfi_probe(struct platform_device *pdev) ret = PTR_ERR(spfi->rx_ch); spfi->rx_ch = NULL; if (ret == -EPROBE_DEFER) - goto disable_pm; + goto free_dma; } if (!spfi->tx_ch || !spfi->rx_ch) { @@ -647,6 +647,7 @@ static int img_spfi_probe(struct platform_device *pdev) disable_pm: pm_runtime_disable(spfi->dev); +free_dma: if (spfi->rx_ch) dma_release_channel(spfi->rx_ch); if (spfi->tx_ch) diff --git a/drivers/spi/spi-oc-tiny.c b/drivers/spi/spi-oc-tiny.c index 29333b1f82d7a0..1cd2a934c03292 100644 --- a/drivers/spi/spi-oc-tiny.c +++ b/drivers/spi/spi-oc-tiny.c @@ -210,11 +210,11 @@ static int tiny_spi_probe(struct platform_device *pdev) struct tiny_spi_platform_data *platp = dev_get_platdata(&pdev->dev); struct tiny_spi *hw; struct spi_controller *host; - int err = -ENODEV; + int err; - host = spi_alloc_host(&pdev->dev, sizeof(struct tiny_spi)); + host = devm_spi_alloc_host(&pdev->dev, sizeof(struct tiny_spi)); if (!host) - return err; + return -ENOMEM; /* setup the host state. */ host->bus_num = pdev->id; @@ -232,10 +232,8 @@ static int tiny_spi_probe(struct platform_device *pdev) /* find and map our resources */ hw->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(hw->base)) { - err = PTR_ERR(hw->base); - goto exit; - } + if (IS_ERR(hw->base)) + return PTR_ERR(hw->base); /* irq is optional */ hw->irq = platform_get_irq(pdev, 0); if (hw->irq >= 0) { @@ -243,7 +241,7 @@ static int tiny_spi_probe(struct platform_device *pdev) err = devm_request_irq(&pdev->dev, hw->irq, tiny_spi_irq, 0, pdev->name, hw); if (err) - goto exit; + return err; } /* find platform data */ if (platp) { @@ -252,29 +250,23 @@ static int tiny_spi_probe(struct platform_device *pdev) } else { err = tiny_spi_of_probe(pdev); if (err) - goto exit; + return err; } /* register our spi controller */ err = spi_bitbang_start(&hw->bitbang); if (err) - goto exit; + return err; dev_info(&pdev->dev, "base %p, irq %d\n", hw->base, hw->irq); return 0; - -exit: - spi_controller_put(host); - return err; } static void tiny_spi_remove(struct platform_device *pdev) { struct tiny_spi *hw = platform_get_drvdata(pdev); - struct spi_controller *host = hw->bitbang.ctlr; spi_bitbang_stop(&hw->bitbang); - spi_controller_put(host); } #ifdef CONFIG_OF diff --git a/drivers/spi/spi-sprd-adi.c b/drivers/spi/spi-sprd-adi.c index e7d83c16b46c05..a6baaf02b3c86e 100644 --- a/drivers/spi/spi-sprd-adi.c +++ b/drivers/spi/spi-sprd-adi.c @@ -553,6 +553,12 @@ static int sprd_adi_probe(struct platform_device *pdev) return -ENXIO; } else { switch (ret) { + case 0: + /* + * Only reachable with CONFIG_HWSPINLOCK=n, where the + * of_hwspin_lock_get_id() stub returns 0. + */ + fallthrough; case -ENOENT: dev_info(&pdev->dev, "no hardware spinlock supplied\n"); break; diff --git a/drivers/staging/fbtft/fbtft-sysfs.c b/drivers/staging/fbtft/fbtft-sysfs.c index d05599d80011a1..343545e83a3772 100644 --- a/drivers/staging/fbtft/fbtft-sysfs.c +++ b/drivers/staging/fbtft/fbtft-sysfs.c @@ -98,7 +98,7 @@ sprintf_gamma(struct fbtft_par *par, u32 *curves, char *buf) mutex_lock(&par->gamma.lock); for (i = 0; i < par->gamma.num_curves; i++) { for (j = 0; j < par->gamma.num_values; j++) - len += scnprintf(&buf[len], PAGE_SIZE, + len += sysfs_emit_at(buf, len, "%04x ", curves[i * par->gamma.num_values + j]); buf[len - 1] = '\n'; } diff --git a/drivers/staging/greybus/audio_gb.c b/drivers/staging/greybus/audio_gb.c index 9d8994fdb41a26..144591f1a51280 100644 --- a/drivers/staging/greybus/audio_gb.c +++ b/drivers/staging/greybus/audio_gb.c @@ -37,6 +37,19 @@ int gb_audio_gb_get_topology(struct gb_connection *connection, return ret; } + /* + * The size_* fields are supplied by the module and are used by + * gbaudio_tplg_parse_data() to compute offsets into the blob; make + * sure the sections fit within the fetched topology, so walking it + * cannot read out of bounds. + */ + if ((u64)le32_to_cpu(topo->size_dais) + le32_to_cpu(topo->size_controls) + + le32_to_cpu(topo->size_widgets) + le32_to_cpu(topo->size_routes) > + size - sizeof(*topo)) { + kfree(topo); + return -EINVAL; + } + *topology = topo; return 0; diff --git a/drivers/staging/greybus/audio_manager_sysfs.c b/drivers/staging/greybus/audio_manager_sysfs.c index fcd518f9540cd4..ff323ca8154f3f 100644 --- a/drivers/staging/greybus/audio_manager_sysfs.c +++ b/drivers/staging/greybus/audio_manager_sysfs.c @@ -23,7 +23,7 @@ static ssize_t manager_sysfs_add_store(struct kobject *kobj, desc.name, &desc.vid, &desc.pid, &desc.intf_id, &desc.ip_devices, &desc.op_devices); - if (num != 7) + if (num != 6) return -EINVAL; num = gb_audio_manager_add(&desc); diff --git a/drivers/staging/media/ipu7/ipu7-isys.c b/drivers/staging/media/ipu7/ipu7-isys.c index bf262c01a2b803..601e5a79ef8ece 100644 --- a/drivers/staging/media/ipu7/ipu7-isys.c +++ b/drivers/staging/media/ipu7/ipu7-isys.c @@ -233,6 +233,7 @@ static int isys_notifier_init(struct ipu7_isys *isys) err_parse: fwnode_handle_put(ep); + v4l2_async_nf_cleanup(&isys->notifier); return ret; } diff --git a/drivers/staging/media/ipu7/ipu7.c b/drivers/staging/media/ipu7/ipu7.c index 310e3f24e571fe..48a35bda42370d 100644 --- a/drivers/staging/media/ipu7/ipu7.c +++ b/drivers/staging/media/ipu7/ipu7.c @@ -2343,7 +2343,7 @@ static int ipu7_init_fw_code_region_by_sys(struct ipu7_bus_device *sys, return ret; } - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) { dev_err(dev, "Failed to get runtime PM\n"); return ret; @@ -2702,7 +2702,7 @@ static int ipu7_resume(struct device *dev) if (ret) dev_err(dev, "IPC reset protocol failed!\n"); - ret = pm_runtime_get_sync(&isp->psys->auxdev.dev); + ret = pm_runtime_resume_and_get(&isp->psys->auxdev.dev); if (ret < 0) { dev_err(dev, "Failed to get runtime PM\n"); return 0; diff --git a/drivers/staging/octeon/ethernet-rx.c b/drivers/staging/octeon/ethernet-rx.c index cd36b5ba6f6c29..3e9d58d3215605 100644 --- a/drivers/staging/octeon/ethernet-rx.c +++ b/drivers/staging/octeon/ethernet-rx.c @@ -535,6 +535,8 @@ void cvm_oct_rx_shutdown(struct platform_device *pdev) else cvmx_write_csr(CVMX_POW_WQ_INT_THRX(i), 0); + napi_disable(&plat->rx_group[i].napi); + /* Free the interrupt handler */ free_irq(plat->rx_group[i].irq, &plat->rx_group[i].napi); diff --git a/drivers/staging/octeon/ethernet-tx.c b/drivers/staging/octeon/ethernet-tx.c index 14d10659bce716..785c6492f170bf 100644 --- a/drivers/staging/octeon/ethernet-tx.c +++ b/drivers/staging/octeon/ethernet-tx.c @@ -668,4 +668,6 @@ void cvm_oct_tx_shutdown(void) { /* Free the interrupt handler */ free_irq(OCTEON_IRQ_TIMER1, cvm_oct_device); + + tasklet_kill(&cvm_oct_tx_cleanup_tasklet); } diff --git a/drivers/staging/rtl8723bs/core/rtw_cmd.c b/drivers/staging/rtl8723bs/core/rtw_cmd.c index b932670f5d63ac..a34ee407285be1 100644 --- a/drivers/staging/rtl8723bs/core/rtw_cmd.c +++ b/drivers/staging/rtl8723bs/core/rtw_cmd.c @@ -899,7 +899,7 @@ u8 rtw_setstakey_cmd(struct adapter *padapter, struct sta_info *sta, u8 unicast_ if (enqueue) { ph2c = kzalloc_obj(*ph2c); if (!ph2c) { - kfree(psetstakey_para); + kfree_sensitive(psetstakey_para); res = _FAIL; goto exit; } @@ -907,7 +907,7 @@ u8 rtw_setstakey_cmd(struct adapter *padapter, struct sta_info *sta, u8 unicast_ psetstakey_rsp = kzalloc_obj(*psetstakey_rsp); if (!psetstakey_rsp) { kfree(ph2c); - kfree(psetstakey_para); + kfree_sensitive(psetstakey_para); res = _FAIL; goto exit; } @@ -918,7 +918,7 @@ u8 rtw_setstakey_cmd(struct adapter *padapter, struct sta_info *sta, u8 unicast_ res = rtw_enqueue_cmd(pcmdpriv, ph2c); } else { set_stakey_hdl(padapter, (u8 *)psetstakey_para); - kfree(psetstakey_para); + kfree_sensitive(psetstakey_para); } exit: return res; @@ -958,7 +958,7 @@ u8 rtw_clearstakey_cmd(struct adapter *padapter, struct sta_info *sta, u8 enqueu psetstakey_rsp = kzalloc_obj(*psetstakey_rsp); if (!psetstakey_rsp) { kfree(ph2c); - kfree(psetstakey_para); + kfree_sensitive(psetstakey_para); res = _FAIL; goto exit; } diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c index a9382dc1294b35..4166b1a8eea762 100644 --- a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c +++ b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c @@ -2181,8 +2181,11 @@ _issue_bcn: spin_unlock_bh(&pmlmepriv->bcn_update_lock); - if ((pattrib->pktlen + TXDESC_SIZE) > 512) + if ((pattrib->pktlen + TXDESC_SIZE) > 512) { + rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf); + rtw_free_xmitframe(pxmitpriv, pmgntframe); return; + } pattrib->last_txcmdsz = pattrib->pktlen; @@ -2243,8 +2246,11 @@ void issue_probersp(struct adapter *padapter, unsigned char *da, u8 is_valid_p2p pattrib->pktlen = pattrib->hdrlen; pframe += pattrib->hdrlen; - if (cur_network->ie_length > MAX_IE_SZ) + if (cur_network->ie_length > MAX_IE_SZ) { + rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf); + rtw_free_xmitframe(pxmitpriv, pmgntframe); return; + } if ((pmlmeinfo->state&0x03) == WIFI_FW_AP_STATE) { pwps_ie = rtw_get_wps_ie(cur_network->ies+_FIXED_IE_LENGTH_, cur_network->ie_length-_FIXED_IE_LENGTH_, NULL, &wps_ielen); @@ -2291,8 +2297,11 @@ void issue_probersp(struct adapter *padapter, unsigned char *da, u8 is_valid_p2p u8 *ies = pmgntframe->buf_addr+TXDESC_OFFSET+sizeof(struct ieee80211_hdr_3addr); buf = kzalloc(MAX_IE_SZ, GFP_ATOMIC); - if (!buf) + if (!buf) { + rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf); + rtw_free_xmitframe(pxmitpriv, pmgntframe); return; + } ssid_ie = rtw_get_ie(ies+_FIXED_IE_LENGTH_, WLAN_EID_SSID, &ssid_ielen, (pframe-ies)-_FIXED_IE_LENGTH_); @@ -2670,10 +2679,13 @@ void issue_asocrsp(struct adapter *padapter, unsigned short status, struct sta_i SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq); pmlmeext->mgnt_seq++; - if ((pkt_type == WIFI_ASSOCRSP) || (pkt_type == WIFI_REASSOCRSP)) + if ((pkt_type == WIFI_ASSOCRSP) || (pkt_type == WIFI_REASSOCRSP)) { SetFrameSubType(pwlanhdr, pkt_type); - else + } else { + rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf); + rtw_free_xmitframe(pxmitpriv, pmgntframe); return; + } pattrib->hdrlen = sizeof(struct ieee80211_hdr_3addr); pattrib->pktlen += pattrib->hdrlen; diff --git a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c index c43a0391a5ca73..ee4a9c66aceed4 100644 --- a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c +++ b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c @@ -285,8 +285,8 @@ static struct adapter *rtw_sdio_if1_init(struct dvobj_priv *dvobj, const struct status = _SUCCESS; free_hal_data: - if (status != _SUCCESS && padapter->HalData) - kfree(padapter->HalData); + if (status != _SUCCESS) + rtw_hal_data_deinit(padapter); if (status != _SUCCESS) { rtw_wdev_unregister(padapter->rtw_wdev); diff --git a/drivers/staging/sm750fb/Kconfig b/drivers/staging/sm750fb/Kconfig index 08bcccdd0f1c49..25fe422f55f2c8 100644 --- a/drivers/staging/sm750fb/Kconfig +++ b/drivers/staging/sm750fb/Kconfig @@ -6,6 +6,7 @@ config FB_SM750 select FB_CFB_FILLRECT select FB_CFB_COPYAREA select FB_CFB_IMAGEBLIT + select FB_IOMEM_FOPS help Frame buffer driver for the Silicon Motion SM750 chip with 2D acceleration and dual head support. diff --git a/drivers/staging/sm750fb/sm750.c b/drivers/staging/sm750fb/sm750.c index 89c811e0806c4d..d85ccf5f7a7e51 100644 --- a/drivers/staging/sm750fb/sm750.c +++ b/drivers/staging/sm750fb/sm750.c @@ -261,7 +261,7 @@ static void lynxfb_ops_imageblit(struct fb_info *info, spin_lock(&sm750_dev->slock); sm750_dev->accel.de_imageblit(&sm750_dev->accel, - image->data, image->width >> 3, 0, + image->data, 0, base, pitch, bpp, image->dx, image->dy, image->width, image->height, @@ -591,7 +591,7 @@ static int sm750fb_set_drv(struct lynxfb_par *par) crtc = &par->crtc; crtc->vidmem_size = sm750_dev->vidmem_size; - if (sm750_dev->fb_count > 1) + if (g_dualview) crtc->vidmem_size >>= 1; /* setup crtc and output member */ @@ -896,7 +896,7 @@ static void sm750fb_setup(struct sm750_dev *sm750_dev, char *src) NO_PARAM: if (sm750_dev->revid != SM750LE_REVISION_ID) { - if (sm750_dev->fb_count > 1) { + if (g_dualview) { if (swap) sm750_dev->dataflow = sm750_dual_swap; else diff --git a/drivers/staging/sm750fb/sm750.h b/drivers/staging/sm750fb/sm750.h index d2c522e67f26a5..83229fe155eccc 100644 --- a/drivers/staging/sm750fb/sm750.h +++ b/drivers/staging/sm750fb/sm750.h @@ -73,7 +73,7 @@ struct lynx_accel { u32 rop2); int (*de_imageblit)(struct lynx_accel *accel, const char *p_srcbuf, - u32 src_delta, u32 start_bit, u32 d_base, u32 d_pitch, + u32 start_bit, u32 d_base, u32 d_pitch, u32 byte_per_pixel, u32 dx, u32 dy, u32 width, u32 height, u32 f_color, u32 b_color, u32 rop2); diff --git a/drivers/staging/sm750fb/sm750_accel.c b/drivers/staging/sm750fb/sm750_accel.c index 0316ea69d00978..bac9a209899cfd 100644 --- a/drivers/staging/sm750fb/sm750_accel.c +++ b/drivers/staging/sm750fb/sm750_accel.c @@ -288,8 +288,6 @@ static unsigned int de_get_transparency(struct lynx_accel *accel) * sm750_hw_imageblit * @accel: Acceleration device data * @src_buf: pointer to start of source buffer in system memory - * @src_delta: Pitch value (in bytes) of the source buffer, +ive means top down - * and -ive mean button up * @start_bit: Mono data can start at any bit in a byte, this value should be * 0 to 7 * @dest_base: Address of destination: offset in frame buffer @@ -304,7 +302,7 @@ static unsigned int de_get_transparency(struct lynx_accel *accel) * @rop2: ROP value */ int sm750_hw_imageblit(struct lynx_accel *accel, const char *src_buf, - u32 src_delta, u32 start_bit, u32 dest_base, u32 dest_pitch, + u32 start_bit, u32 dest_base, u32 dest_pitch, u32 byte_per_pixel, u32 dx, u32 dy, u32 width, u32 height, u32 fg_color, u32 bg_color, u32 rop2) { @@ -395,7 +393,7 @@ int sm750_hw_imageblit(struct lynx_accel *accel, const char *src_buf, write_dp_port(accel, *(unsigned int *)remain); } - src_buf += src_delta; + src_buf += bytes_per_scan; } return 0; diff --git a/drivers/staging/sm750fb/sm750_accel.h b/drivers/staging/sm750fb/sm750_accel.h index d15a40cacb84d6..4f609d6af0fa4c 100644 --- a/drivers/staging/sm750fb/sm750_accel.h +++ b/drivers/staging/sm750fb/sm750_accel.h @@ -220,8 +220,6 @@ int sm750_hw_copyarea(struct lynx_accel *accel, /** * sm750_hw_imageblit * @pSrcbuf: pointer to start of source buffer in system memory - * @srcDelta: Pitch value (in bytes) of the source buffer, +ive means top down - *>----- and -ive mean button up * @startBit: Mono data can start at any bit in a byte, this value should be *>----- 0 to 7 * @dBase: Address of destination: offset in frame buffer @@ -236,7 +234,7 @@ int sm750_hw_copyarea(struct lynx_accel *accel, * @rop2: ROP value */ int sm750_hw_imageblit(struct lynx_accel *accel, const char *pSrcbuf, - u32 srcDelta, u32 startBit, u32 dBase, u32 dPitch, + u32 startBit, u32 dBase, u32 dPitch, u32 bytePerPixel, u32 dx, u32 dy, u32 width, u32 height, u32 fColor, u32 bColor, u32 rop2); diff --git a/drivers/thermal/airoha_thermal.c b/drivers/thermal/airoha_thermal.c index b9fd6bfc88e5ea..829a7327fc4035 100644 --- a/drivers/thermal/airoha_thermal.c +++ b/drivers/thermal/airoha_thermal.c @@ -273,7 +273,7 @@ static int airoha_thermal_set_trips(struct thermal_zone_device *tz, int low, if (low != -INT_MAX) { /* Validate low and clamp it to a supported value */ - low = clamp_t(int, high, RAW_TO_TEMP(priv, 0), + low = clamp_t(int, low, RAW_TO_TEMP(priv, 0), RAW_TO_TEMP(priv, FIELD_MAX(EN7581_DOUT_TADC_MASK))); /* We offset the low temp of 1°C to trigger correct event */ @@ -403,7 +403,7 @@ static void airoha_thermal_setup_monitor(struct airoha_thermal_priv *priv) * sen interval is 379 * 52.715us = 19.97ms */ writel(FIELD_PREP(EN7581_FILT_INTERVAL, 1) | - FIELD_PREP(EN7581_FILT_INTERVAL, 379), + FIELD_PREP(EN7581_SEN_INTERVAL, 379), priv->base + EN7581_TEMPMONCTL2); /* AHB poll is set to 146 * 68.64 = 10.02us */ diff --git a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c index d200734625ee86..5d70301d4a3d9c 100644 --- a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c +++ b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c @@ -356,8 +356,10 @@ static void cleanup_odvp(struct int3400_thermal_priv *priv) kfree(priv->odvp_attrs[i].attr.attr.name); } kfree(priv->odvp_attrs); + priv->odvp_attrs = NULL; } kfree(priv->odvp); + priv->odvp = NULL; priv->odvp_count = 0; } @@ -635,7 +637,6 @@ free_notify: acpi_remove_notify_handler(priv->adev->handle, ACPI_DEVICE_NOTIFY, int3400_notify); free_sysfs: - cleanup_odvp(priv); if (!ZERO_OR_NULL_PTR(priv->data_vault)) { device_remove_bin_file(&pdev->dev, &bin_attr_data_vault); kfree(priv->data_vault); @@ -649,6 +650,7 @@ free_rel_misc: acpi_thermal_rel_misc_device_remove(priv->adev->handle); thermal_zone_device_unregister(priv->thermal); free_art_trt: + cleanup_odvp(priv); kfree(priv->trts); kfree(priv->arts); free_priv: diff --git a/drivers/thermal/qcom/qcom-spmi-adc-tm5.c b/drivers/thermal/qcom/qcom-spmi-adc-tm5.c index d7f2e6ca92c2c9..d1b086737bcd28 100644 --- a/drivers/thermal/qcom/qcom-spmi-adc-tm5.c +++ b/drivers/thermal/qcom/qcom-spmi-adc-tm5.c @@ -369,9 +369,6 @@ static int adc_tm5_get_temp(struct thermal_zone_device *tz, int *temp) if (ret < 0) return ret; - if (ret != IIO_VAL_INT) - return -EINVAL; - return 0; } diff --git a/drivers/thermal/renesas/rcar_thermal.c b/drivers/thermal/renesas/rcar_thermal.c index 6e5dcac5d47ae7..fd686da9252e19 100644 --- a/drivers/thermal/renesas/rcar_thermal.c +++ b/drivers/thermal/renesas/rcar_thermal.c @@ -492,12 +492,6 @@ static int rcar_thermal_probe(struct platform_device *pdev) "rcar_thermal", trips, ARRAY_SIZE(trips), priv, &rcar_thermal_zone_ops, NULL, 0, idle); - - ret = thermal_zone_device_enable(priv->zone); - if (ret) { - thermal_zone_device_unregister(priv->zone); - priv->zone = ERR_PTR(ret); - } } if (IS_ERR(priv->zone)) { dev_err(dev, "can't register thermal zone\n"); @@ -506,11 +500,12 @@ static int rcar_thermal_probe(struct platform_device *pdev) goto error_unregister; } - if (chip->use_of_thermal) { + if (chip->use_of_thermal) ret = thermal_add_hwmon_sysfs(priv->zone); - if (ret) - goto error_unregister; - } + else + ret = thermal_zone_device_enable(priv->zone); + if (ret) + goto error_unregister; rcar_thermal_irq_enable(priv); diff --git a/drivers/thermal/spacemit/k1_tsensor.c b/drivers/thermal/spacemit/k1_tsensor.c index 79222d233129dc..a9eaedefdac3b0 100644 --- a/drivers/thermal/spacemit/k1_tsensor.c +++ b/drivers/thermal/spacemit/k1_tsensor.c @@ -156,13 +156,12 @@ static int k1_tsensor_set_trips(struct thermal_zone_device *tz, int low, int hig struct k1_tsensor *ts = ch->ts; u32 val; - if (low >= high) - return -EINVAL; - low = clamp_val(low / 1000 + TEMPERATURE_OFFSET, TEMPERATURE_OFFSET, FIELD_MAX(K1_TSENSOR_THRSH_LOW_MASK)); high = clamp_val(high / 1000 + TEMPERATURE_OFFSET, TEMPERATURE_OFFSET, FIELD_MAX(K1_TSENSOR_THRSH_HIGH_MASK)); + if (low >= high) + return -EINVAL; val = readl(ts->base + K1_TSENSOR_THRSH_REG(ch->id)); diff --git a/drivers/thermal/thermal_hwmon.c b/drivers/thermal/thermal_hwmon.c index ec73d03a1e60a2..45ce21914df869 100644 --- a/drivers/thermal/thermal_hwmon.c +++ b/drivers/thermal/thermal_hwmon.c @@ -95,34 +95,12 @@ thermal_hwmon_lookup_by_type(const struct thermal_zone_device *tz) struct thermal_hwmon_device *hwmon; char type[THERMAL_NAME_LENGTH]; - mutex_lock(&thermal_hwmon_list_lock); list_for_each_entry(hwmon, &thermal_hwmon_list, node) { strscpy(type, tz->type); strreplace(type, '-', '_'); - if (!strcmp(hwmon->type, type)) { - mutex_unlock(&thermal_hwmon_list_lock); + if (!strcmp(hwmon->type, type)) return hwmon; - } } - mutex_unlock(&thermal_hwmon_list_lock); - - return NULL; -} - -/* Find the temperature input matching a given thermal zone */ -static struct thermal_hwmon_temp * -thermal_hwmon_lookup_temp(const struct thermal_hwmon_device *hwmon, - const struct thermal_zone_device *tz) -{ - struct thermal_hwmon_temp *temp; - - mutex_lock(&thermal_hwmon_list_lock); - list_for_each_entry(temp, &hwmon->tz_list, hwmon_node) - if (temp->tz == tz) { - mutex_unlock(&thermal_hwmon_list_lock); - return temp; - } - mutex_unlock(&thermal_hwmon_list_lock); return NULL; } @@ -138,7 +116,9 @@ int thermal_add_hwmon_sysfs(struct thermal_zone_device *tz) struct thermal_hwmon_device *hwmon; struct thermal_hwmon_temp *temp; int new_hwmon_device = 1; - int result; + int result = 0; + + mutex_lock(&thermal_hwmon_list_lock); hwmon = thermal_hwmon_lookup_by_type(tz); if (hwmon) { @@ -147,8 +127,10 @@ int thermal_add_hwmon_sysfs(struct thermal_zone_device *tz) } hwmon = kzalloc_obj(*hwmon); - if (!hwmon) - return -ENOMEM; + if (!hwmon) { + result = -ENOMEM; + goto unlock; + } INIT_LIST_HEAD(&hwmon->tz_list); strscpy(hwmon->type, tz->type, THERMAL_NAME_LENGTH); @@ -196,24 +178,24 @@ int thermal_add_hwmon_sysfs(struct thermal_zone_device *tz) temp->temp_crit_present = true; } - mutex_lock(&thermal_hwmon_list_lock); if (new_hwmon_device) list_add_tail(&hwmon->node, &thermal_hwmon_list); list_add_tail(&temp->hwmon_node, &hwmon->tz_list); - mutex_unlock(&thermal_hwmon_list_lock); - return 0; + goto unlock; - unregister_input: +unregister_input: device_remove_file(hwmon->device, &temp->temp_input.attr); - free_temp_mem: +free_temp_mem: kfree(temp); - unregister_name: +unregister_name: if (new_hwmon_device) hwmon_device_unregister(hwmon->device); - free_mem: +free_mem: if (new_hwmon_device) kfree(hwmon); +unlock: + mutex_unlock(&thermal_hwmon_list_lock); return result; } @@ -221,8 +203,11 @@ EXPORT_SYMBOL_GPL(thermal_add_hwmon_sysfs); void thermal_remove_hwmon_sysfs(struct thermal_zone_device *tz) { + struct thermal_hwmon_temp *temp, *entry; struct thermal_hwmon_device *hwmon; - struct thermal_hwmon_temp *temp; + bool unregister; + + guard(mutex)(&thermal_hwmon_list_lock); hwmon = thermal_hwmon_lookup_by_type(tz); if (unlikely(!hwmon)) { @@ -231,29 +216,25 @@ void thermal_remove_hwmon_sysfs(struct thermal_zone_device *tz) return; } - temp = thermal_hwmon_lookup_temp(hwmon, tz); - if (unlikely(!temp)) { - /* Should never happen... */ - dev_dbg(&tz->device, "temperature input lookup failed!\n"); - return; - } + unregister = hwmon->device->parent == &tz->device; - device_remove_file(hwmon->device, &temp->temp_input.attr); - if (temp->temp_crit_present) - device_remove_file(hwmon->device, &temp->temp_crit.attr); + list_for_each_entry_safe_reverse(temp, entry, &hwmon->tz_list, hwmon_node) { + if (!unregister && temp->tz != tz) + continue; - mutex_lock(&thermal_hwmon_list_lock); - list_del(&temp->hwmon_node); - kfree(temp); - if (!list_empty(&hwmon->tz_list)) { - mutex_unlock(&thermal_hwmon_list_lock); - return; + device_remove_file(hwmon->device, &temp->temp_input.attr); + if (temp->temp_crit_present) + device_remove_file(hwmon->device, &temp->temp_crit.attr); + + list_del(&temp->hwmon_node); + kfree(temp); } - list_del(&hwmon->node); - mutex_unlock(&thermal_hwmon_list_lock); - hwmon_device_unregister(hwmon->device); - kfree(hwmon); + if (unregister) { + list_del(&hwmon->node); + hwmon_device_unregister(hwmon->device); + kfree(hwmon); + } } EXPORT_SYMBOL_GPL(thermal_remove_hwmon_sysfs); diff --git a/drivers/thunderbolt/stream.c b/drivers/thunderbolt/stream.c index 4cc86d8d649183..43d29d69dcdbb8 100644 --- a/drivers/thunderbolt/stream.c +++ b/drivers/thunderbolt/stream.c @@ -512,8 +512,10 @@ tbstream_dev_alloc_tx(struct tbstream_dev *sdev, enum tbstream_frame_pdf pdf, dma_sync_single_for_cpu(dma_dev, sf->frame.buffer_phy, size, DMA_TO_DEVICE); if (pdf == TBSTREAM_DATA) { - if (copy_page_from_iter(sf->page, 0, size, from) != size) + if (copy_page_from_iter(sf->page, 0, size, from) != size) { + sdev->tx_ring.cons--; return ERR_PTR(-EFAULT); + } } else { memset(page_address(sf->page), 0, size); } @@ -671,7 +673,7 @@ tbstream_dev_fops_read_iter(struct kiocb *kiocb, struct iov_iter *to) } nbytes = 0; - while (nbytes < iov_iter_count(to)) { + while (iov_iter_count(to)) { struct tbstream_frame *sf; size_t size, sf_size; @@ -693,7 +695,7 @@ tbstream_dev_fops_read_iter(struct kiocb *kiocb, struct iov_iter *to) } sf_size = tb_ring_frame_size(&sf->frame); - size = min(iov_iter_count(to) - nbytes, sf_size); + size = min(iov_iter_count(to), sf_size); if (copy_page_to_iter(sf->page, sf->offset, size, to) != size) { ret = -EFAULT; @@ -763,10 +765,10 @@ tbstream_dev_fops_write_iter(struct kiocb *kiocb, struct iov_iter *from) } nbytes = 0; - while (nbytes < iov_iter_count(from)) { + while (iov_iter_count(from)) { size_t size; - size = min(iov_iter_count(from) - nbytes, TB_MAX_FRAME_SIZE); + size = min(iov_iter_count(from), TB_MAX_FRAME_SIZE); ret = tbstream_dev_send_data(sdev, from, size); if (ret) { /* diff --git a/drivers/tty/hvc/Kconfig b/drivers/tty/hvc/Kconfig index c2a4e88b328f35..5866195de26a67 100644 --- a/drivers/tty/hvc/Kconfig +++ b/drivers/tty/hvc/Kconfig @@ -79,7 +79,7 @@ config HVC_UDBG config HVC_DCC bool "ARM JTAG DCC console" - depends on ARM || ARM64 + depends on (ARM && (CPU_V6 || CPU_V6K || CPU_V7)) || ARM64 select HVC_DRIVER select SERIAL_CORE_CONSOLE help diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c index 9abaeecd05fc80..c4824c201e1c35 100644 --- a/drivers/tty/serial/amba-pl011.c +++ b/drivers/tty/serial/amba-pl011.c @@ -2543,7 +2543,7 @@ static int pl011_console_setup(struct console *co, char *options) /* Allow pins to be muxed in and configured */ pinctrl_pm_select_default_state(uap->port.dev); - ret = clk_prepare(uap->clk); + ret = clk_prepare_enable(uap->clk); if (ret) return ret; @@ -2572,6 +2572,15 @@ static int pl011_console_setup(struct console *co, char *options) return uart_set_options(&uap->port, co, baud, parity, bits, flow); } +static int pl011_console_exit(struct console *co) +{ + struct uart_amba_port *uap = amba_ports[co->index]; + + clk_disable_unprepare(uap->clk); + + return 0; +} + /** * pl011_console_match - non-standard console matching * @co: registering console @@ -2641,8 +2650,6 @@ pl011_console_write_atomic(struct console *co, struct nbcon_write_context *wctxt if (!nbcon_enter_unsafe(wctxt)) return; - clk_enable(uap->clk); - if (!uap->vendor->always_enabled) { old_cr = pl011_read(uap, REG_CR); pl011_write((old_cr & ~UART011_CR_CTSEN) | (UART01x_CR_UARTEN | UART011_CR_TXE), @@ -2659,8 +2666,6 @@ pl011_console_write_atomic(struct console *co, struct nbcon_write_context *wctxt if (!uap->vendor->always_enabled) pl011_write(old_cr, uap, REG_CR); - clk_disable(uap->clk); - nbcon_exit_unsafe(wctxt); } @@ -2673,8 +2678,6 @@ pl011_console_write_thread(struct console *co, struct nbcon_write_context *wctxt if (!nbcon_enter_unsafe(wctxt)) return; - clk_enable(uap->clk); - if (!uap->vendor->always_enabled) { old_cr = pl011_read(uap, REG_CR); pl011_write((old_cr & ~UART011_CR_CTSEN) | (UART01x_CR_UARTEN | UART011_CR_TXE), @@ -2703,8 +2706,6 @@ pl011_console_write_thread(struct console *co, struct nbcon_write_context *wctxt if (!uap->vendor->always_enabled) pl011_write(old_cr, uap, REG_CR); - clk_disable(uap->clk); - nbcon_exit_unsafe(wctxt); } @@ -2725,6 +2726,7 @@ static struct console amba_console = { .name = "ttyAMA", .device = uart_console_device, .setup = pl011_console_setup, + .exit = pl011_console_exit, .match = pl011_console_match, .write_atomic = pl011_console_write_atomic, .write_thread = pl011_console_write_thread, @@ -3092,21 +3094,45 @@ static void pl011_remove(struct amba_device *dev) static int pl011_suspend(struct device *dev) { struct uart_amba_port *uap = dev_get_drvdata(dev); + int ret; if (!uap) return -EINVAL; - return uart_suspend_port(&amba_reg, &uap->port); + ret = uart_suspend_port(&amba_reg, &uap->port); + if (ret) + return ret; + + if (console_suspend_enabled && uap->port.suspended && + uart_console_registered(&uap->port)) + clk_disable_unprepare(uap->clk); + + return 0; } static int pl011_resume(struct device *dev) { struct uart_amba_port *uap = dev_get_drvdata(dev); + bool resume_console; + int ret; if (!uap) return -EINVAL; - return uart_resume_port(&amba_reg, &uap->port); + resume_console = console_suspend_enabled && + uap->port.suspended && + uart_console_registered(&uap->port); + if (resume_console) { + ret = clk_prepare_enable(uap->clk); + if (ret) + return ret; + } + + ret = uart_resume_port(&amba_reg, &uap->port); + if (ret && resume_console) + clk_disable_unprepare(uap->clk); + + return ret; } #endif diff --git a/drivers/tty/serial/ma35d1_serial.c b/drivers/tty/serial/ma35d1_serial.c index 285b0fe41a86ad..920fe7ff5083b7 100644 --- a/drivers/tty/serial/ma35d1_serial.c +++ b/drivers/tty/serial/ma35d1_serial.c @@ -608,8 +608,14 @@ static int __init ma35d1serial_console_setup(struct console *co, char *options) if (!np || !p) return -ENODEV; - if (of_property_read_u32_array(np, "reg", val32, ARRAY_SIZE(val32)) != 0) + if (of_property_read_u32_array(np, "reg", val32, ARRAY_SIZE(val32)) != 0) { + of_node_put(np); + ma35d1serial_uart_nodes[co->index] = NULL; return -EINVAL; + } + + of_node_put(np); + ma35d1serial_uart_nodes[co->index] = NULL; p->port.iobase = val32[1]; p->port.membase = ioremap(p->port.iobase, MA35_UART_REG_SIZE); @@ -648,8 +654,10 @@ static void ma35d1serial_console_init_port(void) of_node_get(np); ma35d1serial_uart_nodes[i] = np; i++; - if (i == MA35_UART_NR) + if (i == MA35_UART_NR) { + of_node_put(np); break; + } } } } diff --git a/drivers/tty/serial/qcom_geni_serial.c b/drivers/tty/serial/qcom_geni_serial.c index 1e39122ec09f4d..1fc869ad84bd81 100644 --- a/drivers/tty/serial/qcom_geni_serial.c +++ b/drivers/tty/serial/qcom_geni_serial.c @@ -143,6 +143,7 @@ struct qcom_geni_serial_port { unsigned int tx_remaining; unsigned int tx_queued; + bool tx_dma_stale; int wakeup_irq; bool rx_tx_swap; bool cts_rts_swap; @@ -697,6 +698,7 @@ static void qcom_geni_serial_start_tx_dma(struct uart_port *uport) } port->tx_remaining = xmit_size; + port->tx_dma_stale = false; } static void qcom_geni_serial_start_tx_fifo(struct uart_port *uport) @@ -1029,6 +1031,7 @@ static void qcom_geni_serial_handle_tx_dma(struct uart_port *uport) struct qcom_geni_serial_port *port = to_dev_port(uport); struct tty_port *tport = &uport->state->port; unsigned int fifo_len = kfifo_len(&tport->xmit_fifo); + bool tx_dma_stale = port->tx_dma_stale; /* * Only advance the kfifo if it still contains the bytes that were @@ -1039,12 +1042,13 @@ static void qcom_geni_serial_handle_tx_dma(struct uart_port *uport) * kfifo->in, making kfifo_len() wrap to UART_XMIT_SIZE - tx_remaining * and triggering a spurious large DMA transfer of stale data. */ - if (fifo_len >= port->tx_remaining) + if (!tx_dma_stale && fifo_len >= port->tx_remaining) uart_xmit_advance(uport, port->tx_remaining); geni_se_tx_dma_unprep(&port->se, port->tx_dma_addr, port->tx_remaining); port->tx_dma_addr = 0; port->tx_remaining = 0; + port->tx_dma_stale = false; if (!kfifo_is_empty(&tport->xmit_fifo)) qcom_geni_serial_start_tx_dma(uport); @@ -1182,6 +1186,10 @@ static void qcom_geni_serial_shutdown(struct uart_port *uport) static void qcom_geni_serial_flush_buffer_fifo(struct uart_port *uport) { + struct qcom_geni_serial_port *port = to_dev_port(uport); + + if (port->tx_dma_addr) + port->tx_dma_stale = true; qcom_geni_serial_cancel_tx_cmd(uport); } diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index a530ad372b4345..234976fb2a87f3 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -2777,8 +2777,10 @@ int uart_register_driver(struct uart_driver *drv) for (i = 0; i < drv->nr; i++) tty_port_destroy(&drv->state[i].port); tty_driver_kref_put(normal); + drv->tty_driver = NULL; out_kfree: kfree(drv->state); + drv->state = NULL; out: return retval; } @@ -3056,7 +3058,6 @@ static int serial_core_add_one_port(struct uart_driver *drv, struct uart_port *u struct uart_state *state; struct tty_port *port; struct device *tty_dev; - int num_groups; if (uport->line >= drv->nr) return -EINVAL; @@ -3068,6 +3069,22 @@ static int serial_core_add_one_port(struct uart_driver *drv, struct uart_port *u if (state->uart_port) return -EINVAL; + uport->name = kasprintf(GFP_KERNEL, "%s%u", drv->dev_name, + drv->tty_driver->name_base + uport->line); + if (!uport->name) + return -ENOMEM; + + /* + * uart_configure_port() may set uport->attr_group and register the + * console. Allocate room for both groups and a NULL terminator first. + */ + uport->tty_groups = kzalloc_objs(*uport->tty_groups, 3); + if (!uport->tty_groups) { + kfree(uport->name); + return -ENOMEM; + } + uport->tty_groups[0] = &tty_dev_attr_group; + /* Link the port to the driver state table and vice versa */ atomic_set(&state->refcount, 1); init_waitqueue_head(&state->remove_wait); @@ -3084,10 +3101,6 @@ static int serial_core_add_one_port(struct uart_driver *drv, struct uart_port *u state->pm_state = UART_PM_STATE_UNDEFINED; uart_port_set_cons(uport, drv->cons); uport->minor = drv->tty_driver->minor_start + uport->line; - uport->name = kasprintf(GFP_KERNEL, "%s%u", drv->dev_name, - drv->tty_driver->name_base + uport->line); - if (!uport->name) - return -ENOMEM; if (uport->cons && uport->dev) of_console_check(uport->dev->of_node, uport->cons->name, uport->line); @@ -3102,15 +3115,6 @@ static int serial_core_add_one_port(struct uart_driver *drv, struct uart_port *u port->console = uart_console(uport); - num_groups = 2; - if (uport->attr_group) - num_groups++; - - uport->tty_groups = kzalloc_objs(*uport->tty_groups, num_groups); - if (!uport->tty_groups) - return -ENOMEM; - - uport->tty_groups[0] = &tty_dev_attr_group; if (uport->attr_group) uport->tty_groups[1] = uport->attr_group; diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index 6b283fd03ff824..4889076b975f3c 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -3167,8 +3167,10 @@ static int tty_cdev_add(struct tty_driver *driver, dev_t dev, driver->cdevs[index]->ops = &tty_fops; driver->cdevs[index]->owner = driver->owner; err = cdev_add(driver->cdevs[index], dev, count); - if (err) + if (err) { kobject_put(&driver->cdevs[index]->kobj); + driver->cdevs[index] = NULL; + } return err; } @@ -3305,7 +3307,7 @@ EXPORT_SYMBOL_GPL(tty_register_device_attr); void tty_unregister_device(struct tty_driver *driver, unsigned index) { device_destroy(&tty_class, MKDEV(driver->major, driver->minor_start) + index); - if (!(driver->flags & TTY_DRIVER_DYNAMIC_ALLOC)) { + if (!(driver->flags & TTY_DRIVER_DYNAMIC_ALLOC) && driver->cdevs[index]) { cdev_del(driver->cdevs[index]); driver->cdevs[index] = NULL; } diff --git a/drivers/ufs/core/ufs-debugfs.c b/drivers/ufs/core/ufs-debugfs.c index e3dd81d6fe828e..be527209540d74 100644 --- a/drivers/ufs/core/ufs-debugfs.c +++ b/drivers/ufs/core/ufs-debugfs.c @@ -165,7 +165,7 @@ static ssize_t ufs_saved_err_write(struct file *file, const char __user *buf, char val_str[16] = { }; int val, ret; - if (count > sizeof(val_str)) + if (count >= sizeof(val_str)) return -EINVAL; if (copy_from_user(val_str, buf, count)) return -EFAULT; diff --git a/drivers/ufs/core/ufs-rpmb.c b/drivers/ufs/core/ufs-rpmb.c index 62120dc2e9da79..3a0888eaa2ce7f 100644 --- a/drivers/ufs/core/ufs-rpmb.c +++ b/drivers/ufs/core/ufs-rpmb.c @@ -69,6 +69,10 @@ static int ufs_rpmb_route_frames(struct device *dev, u8 *req, unsigned int req_l hba = ufs_rpmb->hba; + /* req_resp is at the end of an RPMB frame. */ + if (req_len < sizeof(*frm_out)) + return -EINVAL; + req_type = be16_to_cpu(frm_out->req_resp); switch (req_type) { diff --git a/drivers/ufs/core/ufs-txeq.c b/drivers/ufs/core/ufs-txeq.c index aa64f2bf4f1efc..7df3d3b18cbec7 100644 --- a/drivers/ufs/core/ufs-txeq.c +++ b/drivers/ufs/core/ufs-txeq.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include "ufshcd-priv.h" @@ -1216,14 +1217,25 @@ static int ufshcd_tx_eqtr(struct ufs_hba *hba, struct ufs_pa_layer_attr *pwr_mode) { struct ufs_pa_layer_attr old_pwr_info; + unsigned int noio_flag; int ret; + /* + * ufshcd_tx_eqtr() is called from a power-mode-change context where + * I/O is suspended. Use memalloc_noio_save() to propagate GFP_NOIO + * to all allocations in the call tree instead of tagging each call + * site individually. + */ + noio_flag = memalloc_noio_save(); + if (!params->eqtr_record) { params->eqtr_record = devm_kzalloc(hba->dev, sizeof(*params->eqtr_record), GFP_KERNEL); - if (!params->eqtr_record) - return -ENOMEM; + if (!params->eqtr_record) { + ret = -ENOMEM; + goto out_noio_restore; + } } memcpy(&old_pwr_info, &hba->pwr_info, sizeof(struct ufs_pa_layer_attr)); @@ -1231,23 +1243,26 @@ static int ufshcd_tx_eqtr(struct ufs_hba *hba, ret = ufshcd_tx_eqtr_prepare(hba, pwr_mode); if (ret) { dev_err(hba->dev, "Failed to prepare TX EQTR: %d\n", ret); - goto out; + goto out_unprepare; } ret = ufshcd_vops_tx_eqtr_notify(hba, PRE_CHANGE, pwr_mode); if (ret) - goto out; + goto out_unprepare; ret = __ufshcd_tx_eqtr(hba, params, pwr_mode); if (ret) - goto out; + goto out_unprepare; ret = ufshcd_vops_tx_eqtr_notify(hba, POST_CHANGE, pwr_mode); -out: +out_unprepare: if (ret) ufshcd_tx_eqtr_unprepare(hba, &old_pwr_info); +out_noio_restore: + memalloc_noio_restore(noio_flag); + return ret; } diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index a41b56dbf3b7fa..86f49c0522ec05 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -1317,6 +1317,7 @@ static int ufshcd_wait_for_pending_cmds(struct ufs_hba *hba, break; } + __set_current_state(TASK_UNINTERRUPTIBLE); io_schedule_timeout(msecs_to_jiffies(20)); if (ktime_to_us(ktime_sub(ktime_get(), start)) > wait_timeout_us) { @@ -3865,7 +3866,7 @@ int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, u8 **buf, enum u { struct uc_string_id *uc_str; u8 *str; - int ret; + int ret, uc_len; if (!buf) return -EINVAL; @@ -3890,11 +3891,19 @@ int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, u8 **buf, enum u goto out; } + uc_len = uc_str->len - QUERY_DESC_HDR_SIZE; + if (uc_len % sizeof(*uc_str->uc)) { + dev_err(hba->dev, "String Desc has an odd UTF-16 payload length\n"); + str = NULL; + ret = -EINVAL; + goto out; + } + if (fmt == SD_ASCII_STD) { ssize_t ascii_len; int i; - /* remove header and divide by 2 to move from UTF16 to UTF8 */ - ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1; + /* Allow up to three UTF-8 bytes per UTF-16 code unit plus a NUL. */ + ascii_len = uc_len / sizeof(*uc_str->uc) * 3 + 1; str = kzalloc(ascii_len, GFP_KERNEL); if (!str) { ret = -ENOMEM; @@ -3906,7 +3915,7 @@ int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, u8 **buf, enum u * we need to convert to utf-8 so it can be displayed */ ret = utf16s_to_utf8s(uc_str->uc, - uc_str->len - QUERY_DESC_HDR_SIZE, + uc_len / sizeof(*uc_str->uc), UTF16_BIG_ENDIAN, str, ascii_len - 1); /* replace non-printable or non-ASCII characters with spaces */ @@ -3916,11 +3925,17 @@ int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, u8 **buf, enum u str[ret++] = '\0'; } else { - str = kmemdup(uc_str->uc, uc_str->len, GFP_KERNEL); + /* + * Keep the bLength-sized raw output for the RPMB device ID ABI. + * The two bytes beyond the UTF-16 payload are explicitly zeroed + * instead of being read past the descriptor buffer. + */ + str = kzalloc(uc_str->len, GFP_KERNEL); if (!str) { ret = -ENOMEM; goto out; } + memcpy(str, uc_str->uc, uc_len); ret = uc_str->len; } out: @@ -4715,7 +4730,9 @@ static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba) ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), &pwr_info->lane_tx); - if (!pwr_info->lane_rx || !pwr_info->lane_tx) { + if (!pwr_info->lane_rx || !pwr_info->lane_tx || + pwr_info->lane_rx > UFS_MAX_LANES || + pwr_info->lane_tx > UFS_MAX_LANES) { dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n", __func__, pwr_info->lane_rx, @@ -5846,8 +5863,8 @@ void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); enum utp_ocs ocs; - if (WARN_ONCE(!cmd, "cqe->command_desc_base_addr = %#llx\n", - le64_to_cpu(cqe->command_desc_base_addr))) + if (WARN_ONCE(!cmd, "invalid completion tag %d, cqe->command_desc_base_addr = %#llx\n", + task_tag, cqe ? le64_to_cpu(cqe->command_desc_base_addr) : 0ULL)) return; if (hba->monitor.enabled) { diff --git a/drivers/uio/uio.c b/drivers/uio/uio.c index 1e4ade78ed8491..e77d5e7d5f64cf 100644 --- a/drivers/uio/uio.c +++ b/drivers/uio/uio.c @@ -1057,6 +1057,11 @@ int __uio_register_device(struct module *owner, err_request_irq: uio_dev_del_attributes(idev); err_uio_dev_add_attributes: + mutex_lock(&idev->info_lock); + idev->info = NULL; + mutex_unlock(&idev->info_lock); + wake_up_interruptible(&idev->wait); + kill_fasync(&idev->async_queue, SIGIO, POLL_HUP); device_del(&idev->dev); err_device_create: uio_free_minor(idev->minor); diff --git a/drivers/usb/atm/usbatm.c b/drivers/usb/atm/usbatm.c index 9600e1ec099304..7b0c791399eaac 100644 --- a/drivers/usb/atm/usbatm.c +++ b/drivers/usb/atm/usbatm.c @@ -917,8 +917,8 @@ static int usbatm_atm_init(struct usbatm_data *instance) instance->atm_dev = atm_dev; - atm_dev->ci_range.vpi_bits = ATM_CI_MAX; - atm_dev->ci_range.vci_bits = ATM_CI_MAX; + atm_dev->ci_range.vpi_bits = 8; + atm_dev->ci_range.vci_bits = 16; atm_dev->signal = ATM_PHY_SIG_UNKNOWN; /* temp init ATM device, set to 128kbit */ diff --git a/drivers/usb/chipidea/ci_hdrc_imx.c b/drivers/usb/chipidea/ci_hdrc_imx.c index 56d2ba824a0b20..282314eea7fc1a 100644 --- a/drivers/usb/chipidea/ci_hdrc_imx.c +++ b/drivers/usb/chipidea/ci_hdrc_imx.c @@ -528,7 +528,7 @@ static int ci_hdrc_imx_probe(struct platform_device *pdev) if (data->wakeup_irq > 0) { irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s:wakeup", pdata.name); if (!irq_name) { - dev_err_probe(dev, -ENOMEM, "failed to create irq_name\n"); + ret = dev_err_probe(dev, -ENOMEM, "failed to create irq_name\n"); goto err_clk; } diff --git a/drivers/usb/core/driver.c b/drivers/usb/core/driver.c index f63004417058e6..7f33fe5ba03bde 100644 --- a/drivers/usb/core/driver.c +++ b/drivers/usb/core/driver.c @@ -228,14 +228,16 @@ static void usb_free_dynids(struct usb_driver *usb_drv) } static const struct usb_device_id *usb_match_dynamic_id(struct usb_interface *intf, - const struct usb_driver *drv) + const struct usb_driver *drv, + struct usb_device_id *id_copy) { struct usb_dynid *dynid; guard(mutex)(&usb_dynids_lock); list_for_each_entry(dynid, &drv->dynids.list, node) { if (usb_match_one_id(intf, &dynid->id)) { - return &dynid->id; + *id_copy = dynid->id; + return id_copy; } } return NULL; @@ -321,6 +323,7 @@ static int usb_probe_interface(struct device *dev) struct usb_interface *intf = to_usb_interface(dev); struct usb_device *udev = interface_to_usbdev(intf); const struct usb_device_id *id; + struct usb_device_id id_copy; int error = -ENODEV; int lpm_disable_error = -ENODEV; @@ -340,7 +343,7 @@ static int usb_probe_interface(struct device *dev) return error; } - id = usb_match_dynamic_id(intf, driver); + id = usb_match_dynamic_id(intf, driver, &id_copy); if (!id) id = usb_match_id(intf, driver->id_table); if (!id) @@ -892,6 +895,7 @@ static int usb_device_match(struct device *dev, const struct device_driver *drv) struct usb_interface *intf; const struct usb_driver *usb_drv; const struct usb_device_id *id; + struct usb_device_id id_copy; /* device drivers never match interfaces */ if (is_usb_device_driver(drv)) @@ -904,7 +908,7 @@ static int usb_device_match(struct device *dev, const struct device_driver *drv) if (id) return 1; - id = usb_match_dynamic_id(intf, usb_drv); + id = usb_match_dynamic_id(intf, usb_drv, &id_copy); if (id) return 1; } diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c index 183a25f65ac897..dd6d6b11199e05 100644 --- a/drivers/usb/gadget/configfs.c +++ b/drivers/usb/gadget/configfs.c @@ -1177,7 +1177,7 @@ static ssize_t os_desc_qw_sign_show(struct config_item *item, char *page) struct gadget_info *gi = os_desc_item_to_gadget_info(item); int res; - res = utf16s_to_utf8s((wchar_t *) gi->qw_sign, OS_STRING_QW_SIGN_LEN, + res = utf16s_to_utf8s((wchar_t *) gi->qw_sign, OS_STRING_QW_SIGN_LEN / 2, UTF16_LITTLE_ENDIAN, page, PAGE_SIZE - 1); page[res++] = '\n'; @@ -1199,7 +1199,7 @@ static ssize_t os_desc_qw_sign_store(struct config_item *item, const char *page, mutex_lock(&gi->lock); res = utf8s_to_utf16s(page, l, UTF16_LITTLE_ENDIAN, (wchar_t *) gi->qw_sign, - OS_STRING_QW_SIGN_LEN); + OS_STRING_QW_SIGN_LEN / 2); if (res > 0) res = len; mutex_unlock(&gi->lock); diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c index 4ec37c1fdd22e0..563390f05f660f 100644 --- a/drivers/usb/gadget/function/f_fs.c +++ b/drivers/usb/gadget/function/f_fs.c @@ -1705,13 +1705,13 @@ static int ffs_dmabuf_transfer(struct file *file, /* In the meantime, endpoint got disabled or changed. */ if (epfile->ep != ep) { ret = -ESHUTDOWN; - goto err_fence_put; + goto err_fence_free; } usb_req = usb_ep_alloc_request(ep->ep, GFP_ATOMIC); if (!usb_req) { ret = -ENOMEM; - goto err_fence_put; + goto err_fence_free; } /* @@ -1760,9 +1760,9 @@ static int ffs_dmabuf_transfer(struct file *file, return ret; -err_fence_put: +err_fence_free: spin_unlock_irq(&epfile->ffs->eps_lock); - dma_fence_put(&fence->base); + kfree(fence); err_resv_unlock: dma_resv_unlock(dmabuf->resv); err_attachment_put: diff --git a/drivers/usb/gadget/function/f_uac1.c b/drivers/usb/gadget/function/f_uac1.c index 85c502e98f5774..7a81cd176abd33 100644 --- a/drivers/usb/gadget/function/f_uac1.c +++ b/drivers/usb/gadget/function/f_uac1.c @@ -1594,7 +1594,8 @@ static ssize_t f_uac1_opts_##name##_store(struct config_item *item, \ const char *page, size_t len) \ { \ struct f_uac1_opts *opts = to_f_uac1_opts(item); \ - char *split_page = NULL; \ + char *buf = NULL; \ + char *split_page; \ int ret = -EINVAL; \ char *token; \ u32 num; \ @@ -1608,18 +1609,22 @@ static ssize_t f_uac1_opts_##name##_store(struct config_item *item, \ \ i = 0; \ memset(opts->name##s, 0x00, sizeof(opts->name##s)); \ - split_page = kstrdup(page, GFP_KERNEL); \ + buf = kstrdup(page, GFP_KERNEL); \ + split_page = buf; \ while ((token = strsep(&split_page, ",")) != NULL) { \ ret = kstrtou32(token, 0, &num); \ if (ret) \ goto end; \ - \ + if (i >= UAC_MAX_RATES) { \ + ret = -EINVAL; \ + goto end; \ + } \ opts->name##s[i++] = num; \ ret = len; \ }; \ \ end: \ - kfree(split_page); \ + kfree(buf); \ mutex_unlock(&opts->lock); \ return ret; \ } \ diff --git a/drivers/usb/gadget/function/f_uac1_legacy.c b/drivers/usb/gadget/function/f_uac1_legacy.c index 5d201a2e30e7f1..3f52099a4fdd5b 100644 --- a/drivers/usb/gadget/function/f_uac1_legacy.c +++ b/drivers/usb/gadget/function/f_uac1_legacy.c @@ -888,60 +888,10 @@ UAC1_INT_ATTRIBUTE(req_buf_size); UAC1_INT_ATTRIBUTE(req_count); UAC1_INT_ATTRIBUTE(audio_buf_size); -#define UAC1_STR_ATTRIBUTE(name) \ -static ssize_t f_uac1_opts_##name##_show(struct config_item *item, \ - char *page) \ -{ \ - struct f_uac1_legacy_opts *opts = to_f_uac1_opts(item); \ - int result; \ - \ - mutex_lock(&opts->lock); \ - result = sprintf(page, "%s\n", opts->name); \ - mutex_unlock(&opts->lock); \ - \ - return result; \ -} \ - \ -static ssize_t f_uac1_opts_##name##_store(struct config_item *item, \ - const char *page, size_t len) \ -{ \ - struct f_uac1_legacy_opts *opts = to_f_uac1_opts(item); \ - int ret = -EBUSY; \ - char *tmp; \ - \ - mutex_lock(&opts->lock); \ - if (opts->refcnt) \ - goto end; \ - \ - tmp = kstrndup(page, len, GFP_KERNEL); \ - if (tmp) { \ - ret = -ENOMEM; \ - goto end; \ - } \ - if (opts->name##_alloc) \ - kfree(opts->name); \ - opts->name##_alloc = true; \ - opts->name = tmp; \ - ret = len; \ - \ -end: \ - mutex_unlock(&opts->lock); \ - return ret; \ -} \ - \ -CONFIGFS_ATTR(f_uac1_opts_, name) - -UAC1_STR_ATTRIBUTE(fn_play); -UAC1_STR_ATTRIBUTE(fn_cap); -UAC1_STR_ATTRIBUTE(fn_cntl); - static struct configfs_attribute *f_uac1_attrs[] = { &f_uac1_opts_attr_req_buf_size, &f_uac1_opts_attr_req_count, &f_uac1_opts_attr_audio_buf_size, - &f_uac1_opts_attr_fn_play, - &f_uac1_opts_attr_fn_cap, - &f_uac1_opts_attr_fn_cntl, NULL, }; @@ -956,12 +906,6 @@ static void f_audio_free_inst(struct usb_function_instance *f) struct f_uac1_legacy_opts *opts; opts = container_of(f, struct f_uac1_legacy_opts, func_inst); - if (opts->fn_play_alloc) - kfree(opts->fn_play); - if (opts->fn_cap_alloc) - kfree(opts->fn_cap); - if (opts->fn_cntl_alloc) - kfree(opts->fn_cntl); kfree(opts); } diff --git a/drivers/usb/gadget/function/f_uac2.c b/drivers/usb/gadget/function/f_uac2.c index 897787d0803c1f..d8cf710085a05e 100644 --- a/drivers/usb/gadget/function/f_uac2.c +++ b/drivers/usb/gadget/function/f_uac2.c @@ -2012,7 +2012,8 @@ static ssize_t f_uac2_opts_##name##_store(struct config_item *item, \ const char *page, size_t len) \ { \ struct f_uac2_opts *opts = to_f_uac2_opts(item); \ - char *split_page = NULL; \ + char *buf = NULL; \ + char *split_page; \ int ret = -EINVAL; \ char *token; \ u32 num; \ @@ -2026,18 +2027,22 @@ static ssize_t f_uac2_opts_##name##_store(struct config_item *item, \ \ i = 0; \ memset(opts->name##s, 0x00, sizeof(opts->name##s)); \ - split_page = kstrdup(page, GFP_KERNEL); \ + buf = kstrdup(page, GFP_KERNEL); \ + split_page = buf; \ while ((token = strsep(&split_page, ",")) != NULL) { \ ret = kstrtou32(token, 0, &num); \ if (ret) \ goto end; \ - \ + if (i >= UAC_MAX_RATES) { \ + ret = -EINVAL; \ + goto end; \ + } \ opts->name##s[i++] = num; \ ret = len; \ }; \ \ end: \ - kfree(split_page); \ + kfree(buf); \ mutex_unlock(&opts->lock); \ return ret; \ } \ diff --git a/drivers/usb/gadget/function/u_uac1_legacy.h b/drivers/usb/gadget/function/u_uac1_legacy.h index b5df9bcbbeba7e..b9ddae550ff3c2 100644 --- a/drivers/usb/gadget/function/u_uac1_legacy.h +++ b/drivers/usb/gadget/function/u_uac1_legacy.h @@ -62,9 +62,6 @@ struct f_uac1_legacy_opts { char *fn_cap; char *fn_cntl; unsigned bound:1; - unsigned fn_play_alloc:1; - unsigned fn_cap_alloc:1; - unsigned fn_cntl_alloc:1; struct mutex lock; int refcnt; }; diff --git a/drivers/usb/gadget/udc/aspeed_udc.c b/drivers/usb/gadget/udc/aspeed_udc.c index 75f9c831b21a62..54f81e66800945 100644 --- a/drivers/usb/gadget/udc/aspeed_udc.c +++ b/drivers/usb/gadget/udc/aspeed_udc.c @@ -1431,25 +1431,12 @@ static void ast_udc_init_hw(struct ast_udc_dev *udc) ast_udc_write(udc, 0, AST_UDC_EP0_CTRL); } -static void ast_udc_remove(struct platform_device *pdev) +static void ast_udc_cleanup(struct platform_device *pdev) { struct ast_udc_dev *udc = platform_get_drvdata(pdev); unsigned long flags; u32 ctrl; - usb_del_gadget_udc(&udc->gadget); - if (udc->driver) { - /* - * This is broken as only some cleanup is skipped, *udev is - * freed and the register mapping goes away. Any further usage - * probably crashes. Also the device is unbound, so the skipped - * cleanup is never catched up later. - */ - dev_alert(&pdev->dev, - "Driver is busy and still going away. Fasten your seat belts!\n"); - return; - } - spin_lock_irqsave(&udc->lock, flags); /* Disable upstream port connection */ @@ -1469,6 +1456,26 @@ static void ast_udc_remove(struct platform_device *pdev) udc->ep0_buf = NULL; } +static void ast_udc_remove(struct platform_device *pdev) +{ + struct ast_udc_dev *udc = platform_get_drvdata(pdev); + + usb_del_gadget_udc(&udc->gadget); + if (udc->driver) { + /* + * This is broken as only some cleanup is skipped, *udev is + * freed and the register mapping goes away. Any further usage + * probably crashes. Also the device is unbound, so the skipped + * cleanup is never catched up later. + */ + dev_alert(&pdev->dev, + "Driver is busy and still going away. Fasten your seat belts!\n"); + return; + } + + ast_udc_cleanup(pdev); +} + static int ast_udc_probe(struct platform_device *pdev) { enum usb_device_speed max_speed; @@ -1521,6 +1528,12 @@ static int ast_udc_probe(struct platform_device *pdev) AST_UDC_NUM_ENDPOINTS, &udc->ep0_buf_dma, GFP_KERNEL); + if (!udc->ep0_buf) { + clk_disable_unprepare(udc->clk); + rc = -ENOMEM; + goto err; + } + udc->gadget.speed = USB_SPEED_UNKNOWN; udc->gadget.max_speed = USB_SPEED_HIGH; udc->creq = udc->reg + AST_UDC_SETUP0; @@ -1550,20 +1563,20 @@ static int ast_udc_probe(struct platform_device *pdev) udc->irq = platform_get_irq(pdev, 0); if (udc->irq < 0) { rc = udc->irq; - goto err; + goto err_cleanup; } rc = devm_request_irq(&pdev->dev, udc->irq, ast_udc_isr, 0, KBUILD_MODNAME, udc); if (rc) { dev_err(&pdev->dev, "Failed to request interrupt\n"); - goto err; + goto err_cleanup; } rc = usb_add_gadget_udc(&pdev->dev, &udc->gadget); if (rc) { dev_err(&pdev->dev, "Failed to add gadget udc\n"); - goto err; + goto err_cleanup; } dev_info(&pdev->dev, "Initialized udc in USB%s mode\n", @@ -1571,9 +1584,10 @@ static int ast_udc_probe(struct platform_device *pdev) return 0; +err_cleanup: + ast_udc_cleanup(pdev); err: dev_err(&pdev->dev, "Failed to udc probe, rc:0x%x\n", rc); - ast_udc_remove(pdev); return rc; } diff --git a/drivers/usb/gadget/udc/r8a66597-udc.c b/drivers/usb/gadget/udc/r8a66597-udc.c index e7a5d8553c0ea2..d190e16d43fc03 100644 --- a/drivers/usb/gadget/udc/r8a66597-udc.c +++ b/drivers/usb/gadget/udc/r8a66597-udc.c @@ -1951,7 +1951,6 @@ static int r8a66597_probe(struct platform_device *pdev) return 0; err_add_udc: - r8a66597_free_request(&r8a66597->ep[0].ep, r8a66597->ep0_req); clean_up2: if (r8a66597->pdata->on_chip) clk_disable_unprepare(r8a66597->clk); diff --git a/drivers/usb/misc/usb-ljca.c b/drivers/usb/misc/usb-ljca.c index 78e94dd89da551..fcb627b49eac65 100644 --- a/drivers/usb/misc/usb-ljca.c +++ b/drivers/usb/misc/usb-ljca.c @@ -595,6 +595,9 @@ static int ljca_enumerate_gpio(struct ljca_adapter *adap) if (gpio_num > LJCA_MAX_GPIO_NUM) return -EINVAL; + if (desc->bank_num > ARRAY_SIZE(valid_pin)) + return -EINVAL; + /* construct platform data */ gpio_info = kzalloc_obj(*gpio_info); if (!gpio_info) diff --git a/drivers/usb/mtu3/mtu3_core.c b/drivers/usb/mtu3/mtu3_core.c index 66dbfe1705d572..a40bf5bad2d5d5 100644 --- a/drivers/usb/mtu3/mtu3_core.c +++ b/drivers/usb/mtu3/mtu3_core.c @@ -1037,9 +1037,14 @@ int ssusb_gadget_suspend(struct ssusb_mtk *ssusb, pm_message_t msg) if (!mtu->gadget_driver) return 0; - if (mtu->connected) + /* Prevent runtime suspend when active connection exists */ + if (mtu->connected && PMSG_IS_AUTO(msg)) return -EBUSY; + /* Perform soft disconnect for system suspend */ + if (mtu->softconnect && !PMSG_IS_AUTO(msg)) + mtu3_dev_on_off(mtu, 0); + mtu3_dev_suspend(mtu); synchronize_irq(mtu->irq); @@ -1055,5 +1060,9 @@ int ssusb_gadget_resume(struct ssusb_mtk *ssusb, pm_message_t msg) mtu3_dev_resume(mtu); + /* Restore soft connect for system resume */ + if (mtu->softconnect && !PMSG_IS_AUTO(msg)) + mtu3_dev_on_off(mtu, 1); + return 0; } diff --git a/drivers/usb/renesas_usbhs/common.c b/drivers/usb/renesas_usbhs/common.c index 8c93bde4b81673..51d3035f82bedd 100644 --- a/drivers/usb/renesas_usbhs/common.c +++ b/drivers/usb/renesas_usbhs/common.c @@ -813,9 +813,6 @@ static void usbhs_remove(struct platform_device *pdev) flush_delayed_work(&priv->notify_hotplug_work); - usbhs_platform_call(priv, hardware_exit, pdev); - reset_control_assert(priv->rsts); - /* * Explicitly free the IRQ to ensure the interrupt handler is * disabled and synchronized before freeing resources. @@ -832,6 +829,9 @@ static void usbhs_remove(struct platform_device *pdev) if (!usbhs_get_dparam(priv, runtime_pwctrl)) usbhsc_power_ctrl(priv, 0); + usbhs_platform_call(priv, hardware_exit, pdev); + reset_control_assert(priv->rsts); + usbhsc_clk_put(priv); pm_runtime_disable(&pdev->dev); } diff --git a/drivers/usb/typec/ucsi/debugfs.c b/drivers/usb/typec/ucsi/debugfs.c index ff33a5e7c6b0dd..a124105b6226da 100644 --- a/drivers/usb/typec/ucsi/debugfs.c +++ b/drivers/usb/typec/ucsi/debugfs.c @@ -162,6 +162,7 @@ void ucsi_debugfs_unregister(struct ucsi *ucsi) debugfs_remove_recursive(ucsi->debugfs->dentry); kfree(ucsi->debugfs); + ucsi->debugfs = NULL; } void ucsi_debugfs_init(void) diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c index 81d74242c816cc..b3a7712a645073 100644 --- a/drivers/usb/typec/ucsi/ucsi.c +++ b/drivers/usb/typec/ucsi/ucsi.c @@ -2367,6 +2367,8 @@ void ucsi_unregister(struct ucsi *ucsi) cancel_delayed_work_sync(&ucsi->work); cancel_work_sync(&ucsi->resume_work); + ucsi_debugfs_unregister(ucsi); + /* Disable notifications */ ucsi->ops->async_control(ucsi, cmd); diff --git a/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c b/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c index ca1b534cb183d9..57b37fca150c23 100644 --- a/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c +++ b/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c @@ -105,6 +105,7 @@ struct gaokun_ucsi { struct notifier_block nb; u16 version; u8 num_ports; + bool registered; }; /* -------------------------------------------------------------------------- */ @@ -482,8 +483,13 @@ static void gaokun_ucsi_register_worker(struct work_struct *work) } ret = ucsi_register(ucsi); - if (ret) + if (ret) { dev_err_probe(ucsi->dev, ret, "ucsi register failed\n"); + gaokun_ec_unregister_notify(uec->ec, &uec->nb); + return; + } + + uec->registered = true; } static int gaokun_ucsi_probe(struct auxiliary_device *adev, @@ -528,8 +534,11 @@ static void gaokun_ucsi_remove(struct auxiliary_device *adev) int i; disable_delayed_work_sync(&uec->work); - gaokun_ec_unregister_notify(uec->ec, &uec->nb); - ucsi_unregister(uec->ucsi); + if (uec->registered) { + gaokun_ec_unregister_notify(uec->ec, &uec->nb); + ucsi_unregister(uec->ucsi); + } + for (i = 0; i < uec->num_ports; ++i) typec_mux_put(uec->ports[i].typec_mux); diff --git a/drivers/vdpa/mlx5/core/mr.c b/drivers/vdpa/mlx5/core/mr.c index 77a479aeaa85ae..b0c5ff23d022b8 100644 --- a/drivers/vdpa/mlx5/core/mr.c +++ b/drivers/vdpa/mlx5/core/mr.c @@ -481,7 +481,7 @@ static int add_direct_chain(struct mlx5_vdpa_dev *mvdev, return 0; err_alloc: - list_for_each_entry_safe(dmr, n, &mr->head, list) { + list_for_each_entry_safe(dmr, n, &tmp, list) { list_del_init(&dmr->list); unmap_direct_mr(mvdev, dmr); kfree(dmr); diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c index 4d116644851d92..c748fe45116350 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c @@ -233,8 +233,11 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr, kthread_init_work(&vdpasim->work, vdpasim_work_fn); vdpasim->worker = kthread_run_worker(0, "vDPA sim worker: %s", dev_attr->name); - if (IS_ERR(vdpasim->worker)) + if (IS_ERR(vdpasim->worker)) { + ret = PTR_ERR(vdpasim->worker); + vdpasim->worker = NULL; goto err_iommu; + } mutex_init(&vdpasim->mutex); spin_lock_init(&vdpasim->iommu_lock); @@ -746,18 +749,24 @@ static void vdpasim_free(struct vdpa_device *vdpa) struct vdpasim *vdpasim = vdpa_to_sim(vdpa); int i; - kthread_cancel_work_sync(&vdpasim->work); - kthread_destroy_worker(vdpasim->worker); + if (vdpasim->worker) { + kthread_cancel_work_sync(&vdpasim->work); + kthread_destroy_worker(vdpasim->worker); + } - for (i = 0; i < vdpasim->dev_attr.nvqs; i++) { - vringh_kiov_cleanup(&vdpasim->vqs[i].out_iov); - vringh_kiov_cleanup(&vdpasim->vqs[i].in_iov); + if (vdpasim->vqs) { + for (i = 0; i < vdpasim->dev_attr.nvqs; i++) { + vringh_kiov_cleanup(&vdpasim->vqs[i].out_iov); + vringh_kiov_cleanup(&vdpasim->vqs[i].in_iov); + } } vdpasim->dev_attr.free(vdpasim); - for (i = 0; i < vdpasim->dev_attr.nas; i++) - vhost_iotlb_reset(&vdpasim->iommu[i]); + if (vdpasim->iommu) { + for (i = 0; i < vdpasim->dev_attr.nas; i++) + vhost_iotlb_reset(&vdpasim->iommu[i]); + } kfree(vdpasim->iommu); kfree(vdpasim->iommu_pt); kfree(vdpasim->vqs); diff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c index a10ed733f0e3af..9914f3ac69aef9 100644 --- a/drivers/vfio/pci/vfio_pci_config.c +++ b/drivers/vfio/pci/vfio_pci_config.c @@ -1272,6 +1272,7 @@ static int vfio_msi_cap_len(struct vfio_pci_core_device *vdev, u8 pos) ret = init_pci_cap_msi_perm(vdev->msi_perm, len, flags); if (ret) { kfree(vdev->msi_perm); + vdev->msi_perm = NULL; return ret; } diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 3f11a9624b9c03..6a184588ff2344 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -554,6 +554,9 @@ static void vfio_pci_core_map_bars(struct vfio_pci_core_device *vdev) vdev->barmap[bar] = IOMEM_ERR_PTR(-ENODEV); + if (pdev->non_mappable_bars) + continue; + if (!pci_resource_len(pdev, i)) continue; diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index 6949b704166d58..38d9c184082d0f 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -1777,7 +1777,8 @@ static long vhost_net_ioctl(struct file *f, unsigned int ioctl, return -EFAULT; /* Zero the trailing space provided by user-space, if any */ - if (clear_user(argp, size_mul(count - copied, sizeof(u64)))) + if (clear_user(argp + size_mul(copied, sizeof(u64)), + size_mul(count - copied, sizeof(u64)))) return -EFAULT; return 0; case VHOST_SET_FEATURES_ARRAY: diff --git a/drivers/video/backlight/Kconfig b/drivers/video/backlight/Kconfig index 7aa1c4b21111fc..39cb7dfb9c0fc5 100644 --- a/drivers/video/backlight/Kconfig +++ b/drivers/video/backlight/Kconfig @@ -193,7 +193,7 @@ config BACKLIGHT_KTD253 config BACKLIGHT_KTD2801 tristate "Backlight Driver for Kinetic KTD2801" - depends on GPIOLIB || COMPILE_TEST + depends on GPIOLIB select LEDS_EXPRESSWIRE help Say Y to enable the backlight driver for the Kinetic KTD2801 1-wire diff --git a/drivers/video/backlight/aw99706.c b/drivers/video/backlight/aw99706.c index 9ac8ba50a9ea76..c4a66b59ecf139 100644 --- a/drivers/video/backlight/aw99706.c +++ b/drivers/video/backlight/aw99706.c @@ -60,7 +60,7 @@ #define AW99706_MTPLDOSEL_REG 0x1E #define AW99706_MTPRUN_REG 0x1F -#define RESV 0 +#define RESV U32_MAX /* Boost switching frequency table, in Hz */ static const u32 aw99706_sw_freq_tbl[] = { @@ -94,17 +94,19 @@ static int aw99706_dt_property_lookup(const struct aw99706_dt_prop *prop, int i; if (!prop->lookup_tbl) { + if (dt_val > (prop->mask >> __ffs(prop->mask))) + return -EINVAL; *val = dt_val; return 0; } for (i = 0; i < prop->tbl_size; i++) - if (prop->lookup_tbl[i] == dt_val) + if (prop->lookup_tbl[i] != RESV && prop->lookup_tbl[i] == dt_val) break; *val = i; - return i == prop->tbl_size ? -1 : 0; + return i == prop->tbl_size ? -EINVAL : 0; } #define MIN_ILED_MAX 5000 @@ -116,11 +118,14 @@ aw99706_dt_property_iled_max_convert(const struct aw99706_dt_prop *prop, u32 dt_val, u8 *val) { if (dt_val > MAX_ILED_MAX || dt_val < MIN_ILED_MAX) - return -1; + return -EINVAL; + + if ((dt_val - MIN_ILED_MAX) % STEP_ILED_MAX) + return -EINVAL; *val = (dt_val - MIN_ILED_MAX) / STEP_ILED_MAX; - return (dt_val - MIN_ILED_MAX) % STEP_ILED_MAX; + return 0; } static const struct aw99706_dt_prop aw99706_dt_props[] = { diff --git a/drivers/video/fbdev/clps711x-fb.c b/drivers/video/fbdev/clps711x-fb.c index 7a7db7100499ed..6789773b22fbe3 100644 --- a/drivers/video/fbdev/clps711x-fb.c +++ b/drivers/video/fbdev/clps711x-fb.c @@ -329,8 +329,6 @@ static int clps711x_fb_probe(struct platform_device *pdev) return 0; - unregister_framebuffer(info); - out_fb_dealloc_cmap: regmap_update_bits(cfb->syscon, SYSCON_OFFSET, SYSCON1_LCDEN, 0); fb_dealloc_cmap(&info->cmap); diff --git a/drivers/video/fbdev/kyro/fbdev.c b/drivers/video/fbdev/kyro/fbdev.c index d756b3603fa6ac..c2373898882290 100644 --- a/drivers/video/fbdev/kyro/fbdev.c +++ b/drivers/video/fbdev/kyro/fbdev.c @@ -369,6 +369,9 @@ static int kyro_dev_overlay_create(u32 ulWidth, static int kyro_dev_overlay_viewport_set(u32 x, u32 y, u32 ulWidth, u32 ulHeight) { + u32 right; + u32 bottom; + if (deviceInfo.ulOverlayOffset == 0) /* probably haven't called CreateOverlay yet */ return -EINVAL; @@ -378,11 +381,30 @@ static int kyro_dev_overlay_viewport_set(u32 x, u32 y, u32 ulWidth, u32 ulHeight (x < 2 && ulWidth + 2 == 0)) return -EINVAL; + /* + * SetOverlayViewPort() adjusts X coordinates by +2 (left) and +1 + * (right) before packing them into 16-bit register fields. + */ + if (x > U16_MAX - 2 || y > U16_MAX) + return -EINVAL; + + right = x + ulWidth; + bottom = y + ulHeight; + + if (right < x || bottom < y) + return -EINVAL; + + right--; + bottom--; + + if (right > U16_MAX - 1 || bottom > U16_MAX) + return -EINVAL; + /* Stop Ramdac Output */ DisableRamdacOutput(deviceInfo.pSTGReg); SetOverlayViewPort(deviceInfo.pSTGReg, - x, y, x + ulWidth - 1, y + ulHeight - 1); + x, y, right, bottom); EnableOverlayPlane(deviceInfo.pSTGReg); /* Start Ramdac Output */ diff --git a/drivers/video/fbdev/tdfxfb.c b/drivers/video/fbdev/tdfxfb.c index cc6a074f3165ba..9a06cef7569945 100644 --- a/drivers/video/fbdev/tdfxfb.c +++ b/drivers/video/fbdev/tdfxfb.c @@ -1385,7 +1385,7 @@ static int tdfxfb_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (err) return err; - err = pci_enable_device(pdev); + err = pcim_enable_device(pdev); if (err) { printk(KERN_ERR "tdfxfb: Can't enable pdev: %d\n", err); return err; diff --git a/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c b/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c index 32cd038cb79bdb..dbbb2cc0e124a2 100644 --- a/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +++ b/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c @@ -16,54 +16,38 @@ /** * struct arm_cca_token_info - a descriptor for the token buffer. - * @challenge: Pointer to the challenge data - * @challenge_size: Size of the challenge data * @granule: PA of the granule to which the token will be written * @offset: Offset within granule to start of buffer in bytes - * @result: result of rsi_attestation_token_continue operation */ struct arm_cca_token_info { - void *challenge; - unsigned long challenge_size; phys_addr_t granule; unsigned long offset; - unsigned long result; }; -static void arm_cca_attestation_init(void *param) -{ - struct arm_cca_token_info *info; - - info = (struct arm_cca_token_info *)param; - - info->result = rsi_attestation_token_init(info->challenge, - info->challenge_size); -} - /** * arm_cca_attestation_continue - Retrieve the attestation token data. * - * @param: pointer to the arm_cca_token_info + * @info: pointer to the arm_cca_token_info * * Attestation token generation is a long running operation and therefore * the token data may not be retrieved in a single call. Moreover, the * token retrieval operation must be requested on the same CPU on which the * attestation token generation was initialised. - * This helper function is therefore scheduled on the same CPU multiple + * This helper function must therefore be executed on the same CPU multiple * times until the entire token data is retrieved. */ -static void arm_cca_attestation_continue(void *param) +static unsigned long +arm_cca_attestation_continue(struct arm_cca_token_info *info) { + unsigned long ret; unsigned long len; unsigned long size; - struct arm_cca_token_info *info; - - info = (struct arm_cca_token_info *)param; size = RSI_GRANULE_SIZE - info->offset; - info->result = rsi_attestation_token_continue(info->granule, - info->offset, size, &len); + ret = rsi_attestation_token_continue(info->granule, info->offset, size, + &len); info->offset += len; + return ret; } /** @@ -74,8 +58,8 @@ static void arm_cca_attestation_continue(void *param) * * Initialise the attestation token generation using the challenge data * passed in the TSM descriptor. Allocate memory for the attestation token - * and schedule calls to retrieve the attestation token on the same CPU - * on which the attestation token generation was initialised. + * and retrieve the attestation token on the same CPU on which the + * attestation token generation was initialised. * * The challenge data must be at least 32 bytes and no more than 64 bytes. If * less than 64 bytes are provided it will be zero padded to 64 bytes. @@ -85,12 +69,11 @@ static void arm_cca_attestation_continue(void *param) * * %-EINVAL - A parameter was not valid. * * %-ENOMEM - Out of memory. * * %-EFAULT - Failed to get IPA for memory page(s). - * * A negative status code as returned by smp_call_function_single(). */ static int arm_cca_report_new(struct tsm_report *report, void *data) { - int ret; - int cpu; + int ret = 0; + unsigned long rsi_result; long max_size; unsigned long token_size = 0; struct arm_cca_token_info info; @@ -103,37 +86,33 @@ static int arm_cca_report_new(struct tsm_report *report, void *data) /* * The attestation token 'init' and 'continue' calls must be - * performed on the same CPU. smp_call_function_single() is used - * instead of simply calling get_cpu() because of the need to - * allocate outblob based on the returned value from the 'init' - * call and that cannot be done in an atomic context. + * performed on the same CPU, so disable CPU migration around + * those operations. */ - cpu = smp_processor_id(); + migrate_disable(); - info.challenge = desc->inblob; - info.challenge_size = desc->inblob_len; - - ret = smp_call_function_single(cpu, arm_cca_attestation_init, - &info, true); - if (ret) - return ret; - max_size = info.result; - - if (max_size <= 0) - return -EINVAL; + max_size = rsi_attestation_token_init(desc->inblob, desc->inblob_len); + if (max_size <= 0) { + ret = -EINVAL; + goto exit_migrate_enable; + } /* Allocate outblob */ token = kvzalloc(max_size, GFP_KERNEL); - if (!token) - return -ENOMEM; + if (!token) { + ret = -ENOMEM; + goto exit_migrate_enable; + } /* * Since the outblob may not be physically contiguous, use a page * to bounce the buffer from RMM. */ buf = alloc_pages_exact(RSI_GRANULE_SIZE, GFP_KERNEL); - if (!buf) - return -ENOMEM; + if (!buf) { + ret = -ENOMEM; + goto exit_migrate_enable; + } /* Get the PA of the memory page(s) that were allocated */ info.granule = (unsigned long)virt_to_phys(buf); @@ -144,21 +123,15 @@ static int arm_cca_report_new(struct tsm_report *report, void *data) info.offset = 0; do { /* - * Schedule a call to retrieve a sub-granule chunk - * of data per loop iteration. + * Retrieve a sub-granule chunk of data per loop + * iteration. */ - ret = smp_call_function_single(cpu, - arm_cca_attestation_continue, - (void *)&info, true); - if (ret != 0) { - token_size = 0; - goto exit_free_granule_page; - } - } while (info.result == RSI_INCOMPLETE && + rsi_result = arm_cca_attestation_continue(&info); + } while (rsi_result == RSI_INCOMPLETE && info.offset < RSI_GRANULE_SIZE); /* Break out in case of failure */ - if (info.result != RSI_SUCCESS && info.result != RSI_INCOMPLETE) { + if (rsi_result != RSI_SUCCESS && rsi_result != RSI_INCOMPLETE) { ret = -ENXIO; token_size = 0; goto exit_free_granule_page; @@ -173,12 +146,14 @@ static int arm_cca_report_new(struct tsm_report *report, void *data) break; memcpy(&token[token_size], buf, info.offset); token_size += info.offset; - } while (info.result == RSI_INCOMPLETE); + } while (rsi_result == RSI_INCOMPLETE); report->outblob = no_free_ptr(token); exit_free_granule_page: report->outblob_len = token_size; free_pages_exact(buf, RSI_GRANULE_SIZE); +exit_migrate_enable: + migrate_enable(); return ret; } diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index 299fa83be1d5f9..75bb4ffe3b8774 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -401,6 +401,32 @@ static const struct cpumask *virtio_irq_get_affinity(struct device *_d, return dev->config->get_vq_affinity(dev, irq_vec); } +/** + * virtio_device_shutdown - break and reset a device on shutdown + * @dev: the device + * + * Drivers with their own .shutdown method should quiesce their activity and + * then call this to stop the device the way the generic shutdown path does. + */ +void virtio_device_shutdown(struct virtio_device *dev) +{ + /* + * Some devices get wedged if you kick them after they are + * reset. Mark all vqs as broken to make sure we don't. + */ + virtio_break_device(dev); + /* + * Guarantee that any callback will see vq->broken as true. + */ + virtio_synchronize_cbs(dev); + /* + * As IOMMUs are reset on shutdown, this will block device access to memory. + * Some devices get wedged if this happens, so reset to make sure it does not. + */ + dev->config->reset(dev); +} +EXPORT_SYMBOL_GPL(virtio_device_shutdown); + static void virtio_dev_shutdown(struct device *_d) { struct virtio_device *dev = dev_to_virtio(_d); @@ -419,20 +445,7 @@ static void virtio_dev_shutdown(struct device *_d) return; } - /* - * Some devices get wedged if you kick them after they are - * reset. Mark all vqs as broken to make sure we don't. - */ - virtio_break_device(dev); - /* - * Guarantee that any callback will see vq->broken as true. - */ - virtio_synchronize_cbs(dev); - /* - * As IOMMUs are reset on shutdown, this will block device access to memory. - * Some devices get wedged if this happens, so reset to make sure it does not. - */ - dev->config->reset(dev); + virtio_device_shutdown(dev); } static int virtio_dev_num_vf(struct device *dev) diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 581ac799d97495..70450e922d8bb5 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -1095,32 +1096,51 @@ static void remove_common(struct virtio_balloon *vb) vb->vdev->config->del_vqs(vb->vdev); } -static void virtballoon_remove(struct virtio_device *vdev) +/* + * Stop all asynchronous balloon work. The device must still be alive so that + * in-flight requests can drain via the host before it is reset or freed. + */ +static void virtballoon_quiesce(struct virtio_balloon *vb) { - struct virtio_balloon *vb = vdev->priv; + struct virtio_device *vdev = vb->vdev; - if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_REPORTING)) page_reporting_unregister(&vb->pr_dev_info); - if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) unregister_oom_notifier(&vb->oom_nb); - if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) virtio_balloon_unregister_shrinker(vb); + spin_lock_irq(&vb->stop_update_lock); vb->stop_update = true; spin_unlock_irq(&vb->stop_update_lock); cancel_work_sync(&vb->update_balloon_size_work); cancel_work_sync(&vb->update_balloon_stats_work); - if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) { + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) cancel_work_sync(&vb->report_free_page_work); +} + +static void virtballoon_remove(struct virtio_device *vdev) +{ + struct virtio_balloon *vb = vdev->priv; + + virtballoon_quiesce(vb); + + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) destroy_workqueue(vb->balloon_wq); - } remove_common(vb); mutex_destroy(&vb->balloon_lock); kfree(vb); } +static void virtballoon_shutdown(struct virtio_device *vdev) +{ + virtballoon_quiesce(vdev->priv); + virtio_device_shutdown(vdev); +} + #ifdef CONFIG_PM_SLEEP static int virtballoon_freeze(struct virtio_device *vdev) { @@ -1165,6 +1185,11 @@ static int virtballoon_validate(struct virtio_device *vdev) else if (!virtio_has_feature(vdev, VIRTIO_BALLOON_F_PAGE_POISON)) __virtio_clear_bit(vdev, VIRTIO_BALLOON_F_REPORTING); + /* + * Disable indirect descriptors to avoid memory allocation in + * virtqueue_add during page reporting. + */ + __virtio_clear_bit(vdev, VIRTIO_RING_F_INDIRECT_DESC); __virtio_clear_bit(vdev, VIRTIO_F_ACCESS_PLATFORM); return 0; } @@ -1186,6 +1211,7 @@ static struct virtio_driver virtio_balloon_driver = { .validate = virtballoon_validate, .probe = virtballoon_probe, .remove = virtballoon_remove, + .shutdown = virtballoon_shutdown, .config_changed = virtballoon_changed, #ifdef CONFIG_PM_SLEEP .freeze = virtballoon_freeze, diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index 164f480b18a6fe..10371ecbc054cb 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -499,7 +499,7 @@ static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned int nvqs, if (!avq_num) return 0; sprintf(avq->name, "avq.%u", avq->vq_index); - vq = vp_setup_vq(vdev, queue_idx++, vp_modern_avq_done, avq->name, + vq = vp_setup_vq(vdev, avq->vq_index, vp_modern_avq_done, avq->name, false, VIRTIO_MSI_NO_VECTOR, &vp_dev->admin_vq.info); if (IS_ERR(vq)) { diff --git a/drivers/virtio/virtio_rtc_driver.c b/drivers/virtio/virtio_rtc_driver.c index 4419735b0f0dcf..74616ba5be119c 100644 --- a/drivers/virtio/virtio_rtc_driver.c +++ b/drivers/virtio/virtio_rtc_driver.c @@ -574,8 +574,8 @@ static int viortc_msg_xfer(struct viortc_vq *vq, struct viortc_msg *msg, * read requests */ -/** timeout for clock readings, where timeouts are considered non-fatal */ -#define VIORTC_MSG_READ_TIMEOUT secs_to_jiffies(60) +/** timeout for runtime requests, where timeouts are considered non-fatal */ +#define VIORTC_MSG_TIMEOUT secs_to_jiffies(60) /** * viortc_read() - VIRTIO_RTC_REQ_READ wrapper @@ -600,7 +600,7 @@ int viortc_read(struct viortc_dev *viortc, u16 vio_clk_id, u64 *reading) VIORTC_MSG_WRITE(hdl, clock_id, &vio_clk_id); ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), - VIORTC_MSG_READ_TIMEOUT); + VIORTC_MSG_TIMEOUT); if (ret) { dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, ret); @@ -642,7 +642,7 @@ int viortc_read_cross(struct viortc_dev *viortc, u16 vio_clk_id, u8 hw_counter, VIORTC_MSG_WRITE(hdl, hw_counter, &hw_counter); ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), - VIORTC_MSG_READ_TIMEOUT); + VIORTC_MSG_TIMEOUT); if (ret) { dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, ret); @@ -809,7 +809,7 @@ int viortc_read_alarm(struct viortc_dev *viortc, u16 vio_clk_id, VIORTC_MSG_WRITE(hdl, clock_id, &vio_clk_id); ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), - 0); + VIORTC_MSG_TIMEOUT); if (ret) { dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, ret); @@ -858,7 +858,7 @@ int viortc_set_alarm(struct viortc_dev *viortc, u16 vio_clk_id, u64 alarm_time, VIORTC_MSG_WRITE(hdl, flags, &flags); ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), - 0); + VIORTC_MSG_TIMEOUT); if (ret) { dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, ret); @@ -900,7 +900,7 @@ int viortc_set_alarm_enabled(struct viortc_dev *viortc, u16 vio_clk_id, VIORTC_MSG_WRITE(hdl, flags, &flags); ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), - 0); + VIORTC_MSG_TIMEOUT); if (ret) { dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, ret); diff --git a/drivers/w1/masters/ds2482.c b/drivers/w1/masters/ds2482.c index 0069e6f854d7f5..7622c87828442d 100644 --- a/drivers/w1/masters/ds2482.c +++ b/drivers/w1/masters/ds2482.c @@ -310,6 +310,10 @@ static u8 ds2482_w1_triplet(void *data, u8 dbit) mutex_unlock(&pdev->access_lock); + /* On bus error, decode to 3 (no device responded) to abort the search */ + if (status < 0) + status = 3 << 5; + /* Decode the status */ return (status >> 5); } diff --git a/drivers/xen/xenbus/xenbus_probe.c b/drivers/xen/xenbus/xenbus_probe.c index fafb2b84fa5c46..9371f88ac55886 100644 --- a/drivers/xen/xenbus/xenbus_probe.c +++ b/drivers/xen/xenbus/xenbus_probe.c @@ -681,7 +681,8 @@ void xenbus_dev_changed(const char *node, struct xen_bus_type *bus) dev->otherend_id); if (state == XenbusStateInitialising && - (state != dev->state || backend != dev->otherend_id)) { + (state != dev->state || + (dev->otherend && backend != dev->otherend_id))) { /* * State has been reset, assume the old one vanished * and new one needs to be probed. diff --git a/drivers/xen/xenbus/xenbus_xs.c b/drivers/xen/xenbus/xenbus_xs.c index d1cca4acb6f330..62274d377b3aaa 100644 --- a/drivers/xen/xenbus/xenbus_xs.c +++ b/drivers/xen/xenbus/xenbus_xs.c @@ -915,19 +915,27 @@ int xs_init(void) int err; struct task_struct *task; - register_reboot_notifier(&xs_reboot_nb); + err = register_reboot_notifier(&xs_reboot_nb); + if (err) + return err; /* Initialize the shared memory rings to talk to xenstored */ err = xb_init_comms(); if (err) - return err; + goto err_unregister_reboot_notifier; task = kthread_run(xenwatch_thread, NULL, "xenwatch"); - if (IS_ERR(task)) - return PTR_ERR(task); + if (IS_ERR(task)) { + err = PTR_ERR(task); + goto err_unregister_reboot_notifier; + } /* shutdown watches for kexec boot */ xs_reset_watches(); return 0; + +err_unregister_reboot_notifier: + unregister_reboot_notifier(&xs_reboot_nb); + return err; } diff --git a/fs/btrfs/acl.c b/fs/btrfs/acl.c index e55b686fe1ab88..662cdd1cbdef6e 100644 --- a/fs/btrfs/acl.c +++ b/fs/btrfs/acl.c @@ -15,6 +15,7 @@ #include "xattr.h" #include "acl.h" #include "misc.h" +#include "btrfs_inode.h" struct posix_acl *btrfs_get_acl(struct inode *inode, int type, bool rcu) { @@ -107,6 +108,9 @@ int btrfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, struct inode *inode = d_inode(dentry); umode_t old_mode = inode->i_mode; + if (btrfs_root_readonly(BTRFS_I(inode)->root)) + return -EROFS; + if (type == ACL_TYPE_ACCESS && acl) { ret = posix_acl_update_mode(idmap, inode, &inode->i_mode, &acl); diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 8def7abb728f7f..c5bcd3c03d2470 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -2047,6 +2047,11 @@ static int btrfs_reclaim_block_group(struct btrfs_block_group *bg, int *reclaime trace_btrfs_reclaim_block_group(bg); ret = btrfs_relocate_chunk(fs_info, bg->start, false); + if (btrfs_is_zoned(fs_info) && ret == -EAGAIN) { + btrfs_dec_block_group_ro(bg); + btrfs_debug(fs_info, "deferring reclaim of chunk %llu", bg->start); + return ret; + } if (ret) { btrfs_dec_block_group_ro(bg); btrfs_err(fs_info, "error relocating chunk %llu", @@ -2113,7 +2118,8 @@ void btrfs_reclaim_block_groups(struct btrfs_fs_info *fs_info, unsigned int limi spin_unlock(&fs_info->unused_bgs_lock); ret = btrfs_reclaim_block_group(bg, &reclaimed); - if (ret && !READ_ONCE(space_info->periodic_reclaim)) + if ((btrfs_is_zoned(fs_info) && ret == -EAGAIN) || + (ret && !READ_ONCE(space_info->periodic_reclaim))) btrfs_link_bg_list(bg, &retry_list); btrfs_put_block_group(bg); diff --git a/fs/btrfs/defrag.c b/fs/btrfs/defrag.c index f0c6758b7055d3..0697b285e05fb8 100644 --- a/fs/btrfs/defrag.c +++ b/fs/btrfs/defrag.c @@ -1130,20 +1130,15 @@ static_assert(PAGE_ALIGNED(CLUSTER_SIZE)); * * - Extent bits are locked */ -static int defrag_one_locked_target(struct btrfs_inode *inode, - struct defrag_target_range *target, - struct folio **folios, int nr_pages, - struct extent_state **cached_state) +static void defrag_one_locked_target(struct btrfs_inode *inode, + struct defrag_target_range *target, + struct folio **folios, int nr_pages, + struct extent_state **cached_state) { struct btrfs_fs_info *fs_info = inode->root->fs_info; - struct extent_changeset *data_reserved = NULL; const u64 start = target->start; const u64 len = target->len; - int ret = 0; - ret = btrfs_delalloc_reserve_space(inode, &data_reserved, start, len); - if (ret < 0) - return ret; btrfs_clear_extent_bit(&inode->io_tree, start, start + len - 1, EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, cached_state); @@ -1164,10 +1159,6 @@ static int defrag_one_locked_target(struct btrfs_inode *inode, continue; btrfs_folio_clamp_set_dirty(fs_info, folio, start, len); } - btrfs_delalloc_release_extents(inode, len); - extent_changeset_free(data_reserved); - - return ret; } static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, @@ -1183,6 +1174,8 @@ static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, u64 cur = start; const unsigned int nr_pages = ((start + len - 1) >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1; + struct extent_changeset *data_reserved = NULL; + u64 last_defrag_end = start; int ret = 0; ASSERT(nr_pages <= CLUSTER_SIZE / PAGE_SIZE); @@ -1192,6 +1185,22 @@ static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, if (!folios) return -ENOMEM; + /* + * Reserve delalloc space before locking the range and before locking + * and dirtying any folios - otherwise we could deadlock, for example + * after defrag of one range we dirty folios and keep them locked when + * we move to the next range, so reserving delalloc space right before + * each range could trigger flushing of delalloc and deadlock on the + * extent lock or trigger a transaction commit with flushoncommit, which + * can either deadlock on the lock of a folio made dirty in the previous + * range or the extent lock. + */ + ret = btrfs_delalloc_reserve_space(inode, &data_reserved, start, len); + if (ret < 0) { + kfree(folios); + return ret; + } + /* Prepare all pages */ for (int i = 0; cur < start + len && i < nr_pages; i++) { folios[i] = defrag_prepare_one_folio(inode, cur >> PAGE_SHIFT); @@ -1226,10 +1235,11 @@ static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, goto unlock_extent; list_for_each_entry(entry, &target_list, list) { - ret = defrag_one_locked_target(inode, entry, folios, nr_pages, - &cached_state); - if (ret < 0) - break; + defrag_one_locked_target(inode, entry, folios, nr_pages, &cached_state); + if (entry->start > last_defrag_end) + btrfs_delalloc_release_space(inode, data_reserved, last_defrag_end, + entry->start - last_defrag_end, true); + last_defrag_end = entry->start + entry->len; } list_for_each_entry_safe(entry, tmp, &target_list, list) { @@ -1246,6 +1256,12 @@ free_folios: folio_put(folios[i]); } kfree(folios); + btrfs_delalloc_release_extents(inode, len); + if (last_defrag_end < start + len) + btrfs_delalloc_release_space(inode, data_reserved, last_defrag_end, + start + len - last_defrag_end, true); + extent_changeset_free(data_reserved); + return ret; } diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 2f1666d9544e14..de6e23b88b406b 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -271,14 +271,15 @@ int btree_csum_one_bio(struct btrfs_bio *bbio) return -EIO; /* - * If an extent_buffer is marked as EXTENT_BUFFER_ZONED_ZEROOUT, don't - * checksum it but zero-out its content. This is done to preserve - * ordering of I/O without unnecessarily writing out data. + * An extent_buffer marked EXTENT_BUFFER_ZONED_ZEROOUT is written out as + * zeros to preserve ordering of I/O without persisting the now + * unnecessary block. The bio is fed from the shared zero page (see + * write_one_eb()), so there is nothing to checksum here. Crucially, the + * buffer's own content is left intact: it may still be referenced, e.g. + * btrfs_free_tree_block() reads its header to add a delayed reference. */ - if (test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)) { - memzero_extent_buffer(eb, 0, eb->len); + if (test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)) return 0; - } if (WARN_ON_ONCE(found_start != eb->start)) return -EIO; @@ -3314,6 +3315,8 @@ static void invalidate_and_check_btree_folios(struct btrfs_fs_info *fs_info) */ rcu_read_lock(); xa_for_each(&fs_info->buffer_tree, index, eb) { + unsigned int refs; + /* Increase the ref so that the eb won't disappear. */ if (!refcount_inc_not_zero(&eb->refs)) continue; @@ -3323,17 +3326,27 @@ static void invalidate_and_check_btree_folios(struct btrfs_fs_info *fs_info) if (test_bit(EXTENT_BUFFER_READING, &eb->bflags)) wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_READING, TASK_UNINTERRUPTIBLE); + /* + * We hold the spinlock to make sure above + * EXTENT_BUFFER_READING flag is cleared with the held + * ref dropped. + * Or we can hit a race window and lead to false alerts. + */ + spin_lock(&eb->refs_lock); + refs = refcount_read(&eb->refs); + spin_unlock(&eb->refs_lock); + /* * The refs threshold is 2, one held by us at the beginning * of the loop, one for the ownership in the buffer tree. */ - if (unlikely(refcount_read(&eb->refs) > 2 || extent_buffer_under_io(eb))) { + if (unlikely(refs > 2 || extent_buffer_under_io(eb))) { WARN_ON_ONCE(IS_ENABLED(CONFIG_BTRFS_DEBUG)); btrfs_warn(fs_info, "unable to release extent buffer %llu owner %llu gen %llu refs %u flags 0x%lx", eb->start, btrfs_header_owner(eb), btrfs_header_generation(eb), - refcount_read(&eb->refs), eb->bflags); + refs, eb->bflags); } free_extent_buffer(eb); rcu_read_lock(); diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 45c4d4a3b29b17..ae6294fedd2d79 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -2379,14 +2379,17 @@ static struct extent_buffer *find_extent_buffer_nolock( static void end_bbio_meta_write(struct btrfs_bio *bbio) { struct extent_buffer *eb = bbio->private; - struct folio_iter fi; if (bbio->bio.bi_status != BLK_STS_OK) set_btree_ioerr(eb); - bio_for_each_folio_all(fi, &bbio->bio) { - btrfs_meta_folio_clear_writeback(fi.folio, eb); - } + /* + * Clear writeback on the buffer's own folios. The bio may carry the + * shared zero page instead (EXTENT_BUFFER_ZONED_ZEROOUT), so iterate + * the extent buffer folios rather than the bio folios. + */ + for (int i = 0; i < num_extent_folios(eb); i++) + btrfs_meta_folio_clear_writeback(eb->folios[i], eb); buffer_tree_clear_mark(eb, PAGECACHE_TAG_WRITEBACK); clear_and_wake_up_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags); @@ -2427,7 +2430,8 @@ static noinline_for_stack void write_one_eb(struct extent_buffer *eb, struct btrfs_fs_info *fs_info = eb->fs_info; struct btrfs_bio *bbio; - prepare_eb_write(eb); + if (!test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)) + prepare_eb_write(eb); bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES, REQ_OP_WRITE | REQ_META | wbc_to_write_flags(wbc), @@ -2447,8 +2451,21 @@ static noinline_for_stack void write_one_eb(struct extent_buffer *eb, btrfs_meta_folio_set_writeback(folio, eb); if (!folio_test_dirty(folio)) wbc->nr_to_write -= folio_nr_pages(folio); - bio_add_folio_nofail(&bbio->bio, folio, range_len, - offset_in_folio(folio, range_start)); + if (test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)) { + u32 off = 0; + + while (off < range_len) { + u32 add = min_t(u32, PAGE_SIZE, range_len - off); + + bio_add_folio_nofail(&bbio->bio, + page_folio(ZERO_PAGE(0)), + add, 0); + off += add; + } + } else { + bio_add_folio_nofail(&bbio->bio, folio, range_len, + offset_in_folio(folio, range_start)); + } wbc_account_cgroup_owner(wbc, folio, range_len); folio_unlock(folio); } @@ -2496,6 +2513,76 @@ void btrfs_btree_wait_writeback_range(struct btrfs_fs_info *fs_info, u64 start, } } +static int write_meta_extent_buffer(struct btrfs_eb_write_context *ctx, + struct writeback_control *wbc) +{ + struct extent_buffer *eb = ctx->eb; + int ret; + + ret = btrfs_check_meta_write_pointer(eb->fs_info, ctx); + if (ret) + return ret; + + if (!lock_extent_buffer_for_io(eb, wbc)) + return 0; + + /* Implies write in zoned mode. */ + if (ctx->zoned_bg) { + /* Mark the last eb in the block group. */ + btrfs_schedule_zone_finish_bg(ctx->zoned_bg, eb); + ctx->zoned_bg->meta_write_pointer += eb->len; + } + write_one_eb(eb, wbc); + return 0; +} + +/* + * On a zoned filesystem, write out the currently dirty metadata extent buffers + * of @bg. Used to flush the active metadata/system block group before the + * ascending-address walk in btree_writepages(), so that walk can pivot the + * active block group away (finishing it) instead of aborting the commit; see + * the caller for details. + */ +static void flush_active_meta_bg(struct address_space *mapping, + struct writeback_control *wbc, + struct btrfs_eb_write_context *ctx, + struct btrfs_block_group *bg) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host); + unsigned long index = bg->start >> fs_info->nodesize_bits; + unsigned long end = (btrfs_block_group_end(bg) - 1) >> fs_info->nodesize_bits; + struct eb_batch batch; + unsigned int nr_ebs; + + ASSERT(btrfs_is_zoned(fs_info)); + lockdep_assert_held(&fs_info->zoned_meta_io_lock); + + eb_batch_init(&batch); + while (index <= end && + (nr_ebs = buffer_tree_get_ebs_tag(fs_info, &index, end, + PAGECACHE_TAG_DIRTY, &batch))) { + struct extent_buffer *eb; + + while ((eb = eb_batch_next(&batch)) != NULL) { + ctx->eb = eb; + + /* + * If the eb is behind the write pointer (-EBUSY, e.g. + * already being written by someone else) skip it and + * carry on. Only a hole at the write pointer (-EAGAIN) + * stops the flush. The main walk in btree_writepages() + * then deals with it. + */ + if (write_meta_extent_buffer(ctx, wbc) == -EAGAIN) { + eb_batch_release(&batch); + return; + } + } + eb_batch_release(&batch); + cond_resched(); + } +} + int btree_writepages(struct address_space *mapping, struct writeback_control *wbc) { struct btrfs_eb_write_context ctx = { .wbc = wbc }; @@ -2531,6 +2618,22 @@ int btree_writepages(struct address_space *mapping, struct writeback_control *wb else tag = PAGECACHE_TAG_DIRTY; btrfs_zoned_meta_io_lock(fs_info); + + /* + * On a zoned filesystem, flush the currently active metadata/system + * block group(s) first, under this same lock, so the ascending-address + * walk below can pivot the active block group instead of aborting the + * transaction commit with -EAGAIN. + */ + if (btrfs_is_zoned(fs_info) && wbc->sync_mode == WB_SYNC_ALL && + !wbc->for_sync) { + if (fs_info->active_meta_bg) + flush_active_meta_bg(mapping, wbc, &ctx, + fs_info->active_meta_bg); + if (fs_info->active_system_bg) + flush_active_meta_bg(mapping, wbc, &ctx, + fs_info->active_system_bg); + } retry: if (wbc->sync_mode == WB_SYNC_ALL) buffer_tree_tag_for_writeback(fs_info, index, end); @@ -2541,28 +2644,13 @@ retry: while ((eb = eb_batch_next(&batch)) != NULL) { ctx.eb = eb; - ret = btrfs_check_meta_write_pointer(eb->fs_info, &ctx); - if (ret) { - if (ret == -EBUSY) - ret = 0; - - if (ret) { - done = true; - break; - } - continue; - } - - if (!lock_extent_buffer_for_io(eb, wbc)) - continue; - - /* Implies write in zoned mode. */ - if (ctx.zoned_bg) { - /* Mark the last eb in the block group. */ - btrfs_schedule_zone_finish_bg(ctx.zoned_bg, eb); - ctx.zoned_bg->meta_write_pointer += eb->len; + ret = write_meta_extent_buffer(&ctx, wbc); + if (ret == -EBUSY) { + ret = 0; + } else if (ret) { + done = true; + break; } - write_one_eb(eb, wbc); } nr_to_write_done = (wbc->nr_to_write <= 0); eb_batch_release(&batch); @@ -3129,47 +3217,71 @@ static inline void btrfs_release_extent_buffer(struct extent_buffer *eb) kmem_cache_free(extent_buffer_cache, eb); } +/* + * Claim a slot to track an extent buffer in, evicting the coldest tracked buffer + * when the array is full. + * + * Slots fill in order until the array is full. After that a CLOCK (second + * chance) scan advances the hand, clearing one reference bit per step, until + * it lands on an unreferenced slot whose buffer is evicted. Clearing a bit per + * step bounds the scan to BTRFS_INHIBITED_EBS_SLOTS iterations. + */ +static int btrfs_inhibit_claim_slot(struct btrfs_trans_handle *trans) +{ + int slot; + + if (trans->nr_inhibited_ebs < BTRFS_INHIBITED_EBS_SLOTS) + return trans->nr_inhibited_ebs++; + + while (trans->inhibited_ebs_referenced & (1U << trans->inhibited_ebs_hand)) { + trans->inhibited_ebs_referenced &= ~(1U << trans->inhibited_ebs_hand); + trans->inhibited_ebs_hand = + (trans->inhibited_ebs_hand + 1) % BTRFS_INHIBITED_EBS_SLOTS; + } + slot = trans->inhibited_ebs_hand; + trans->inhibited_ebs_hand = (trans->inhibited_ebs_hand + 1) % BTRFS_INHIBITED_EBS_SLOTS; + + atomic_dec(&trans->inhibited_ebs[slot]->writeback_inhibitors); + free_extent_buffer(trans->inhibited_ebs[slot]); + + return slot; +} + /* * Inhibit writeback on buffer during transaction. * * @trans: transaction handle that will own the inhibitor * @eb: extent buffer to inhibit writeback on * - * Attempt to track this extent buffer in the transaction's inhibited set. If - * memory allocation fails, the buffer is simply not tracked. It may be written - * back and need re-COW, which is the original behavior. This is acceptable - * since inhibiting writeback is an optimization. + * Attempt to track this extent buffer in the transaction's inhibited set. When + * the set is full the coldest tracked buffer is evicted instead. An untracked + * buffer may be written back and need re-COW, which is the original behavior. + * This is acceptable since inhibiting writeback is an optimization. */ void btrfs_inhibit_eb_writeback(struct btrfs_trans_handle *trans, struct extent_buffer *eb) { - unsigned long index = eb->start >> trans->fs_info->nodesize_bits; - void *old; + int slot; lockdep_assert_held(&eb->lock); - /* Check if already inhibited by this handle. */ - old = xa_load(&trans->writeback_inhibited_ebs, index); - if (old == eb) - return; - - /* Take reference for the xarray entry. */ - refcount_inc(&eb->refs); - old = xa_store(&trans->writeback_inhibited_ebs, index, eb, GFP_NOFS); - if (xa_is_err(old)) { - /* Allocation failed, just skip inhibiting this buffer. */ - free_extent_buffer(eb); - return; + /* Already tracked: set its reference bit (second chance) and return. */ + for (int i = 0; i < trans->nr_inhibited_ebs; i++) { + if (trans->inhibited_ebs[i] == eb) { + trans->inhibited_ebs_referenced |= 1U << i; + return; + } } - /* Handle replacement of different eb at same index. */ - if (old && old != eb) { - struct extent_buffer *old_eb = old; - - atomic_dec(&old_eb->writeback_inhibitors); - free_extent_buffer(old_eb); - } + slot = btrfs_inhibit_claim_slot(trans); + /* + * Pin the eb while the array holds a raw pointer to it; the counter is + * what lock_extent_buffer_for_io() checks. + */ + refcount_inc(&eb->refs); atomic_inc(&eb->writeback_inhibitors); + trans->inhibited_ebs[slot] = eb; + trans->inhibited_ebs_referenced |= 1U << slot; } /* @@ -3177,14 +3289,13 @@ void btrfs_inhibit_eb_writeback(struct btrfs_trans_handle *trans, struct extent_ */ void btrfs_uninhibit_all_eb_writeback(struct btrfs_trans_handle *trans) { - struct extent_buffer *eb; - unsigned long index; - - xa_for_each(&trans->writeback_inhibited_ebs, index, eb) { - atomic_dec(&eb->writeback_inhibitors); - free_extent_buffer(eb); + for (int i = 0; i < trans->nr_inhibited_ebs; i++) { + atomic_dec(&trans->inhibited_ebs[i]->writeback_inhibitors); + free_extent_buffer(trans->inhibited_ebs[i]); } - xa_destroy(&trans->writeback_inhibited_ebs); + trans->nr_inhibited_ebs = 0; + trans->inhibited_ebs_referenced = 0; + trans->inhibited_ebs_hand = 0; } static struct extent_buffer *__alloc_extent_buffer(struct btrfs_fs_info *fs_info, @@ -3842,12 +3953,31 @@ static int release_extent_buffer(struct extent_buffer *eb) return 0; } -void free_extent_buffer(struct extent_buffer *eb) +static void clear_extent_buffer_reading(struct extent_buffer *eb) +{ + clear_and_wake_up_bit(EXTENT_BUFFER_READING, &eb->bflags); +} + +static void free_extent_buffer_clear_reading(struct extent_buffer *eb, + bool clear_reading) { int refs; + if (!eb) return; + /* + * We want to clear EXTENT_BUFFER_READING flag and decrease refs + * in the same critical section. + * This will make sure invalidate_and_check_btree_folios() won't + * see an eb with EXTENT_BUFFER_READING cleared but refs not yet + * decreased. + */ + if (clear_reading) { + spin_lock(&eb->refs_lock); + clear_extent_buffer_reading(eb); + } + refs = refcount_read(&eb->refs); while (1) { if (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags)) { @@ -3858,11 +3988,16 @@ void free_extent_buffer(struct extent_buffer *eb) } /* Optimization to avoid locking eb->refs_lock. */ - if (atomic_try_cmpxchg(&eb->refs.refs, &refs, refs - 1)) + if (atomic_try_cmpxchg(&eb->refs.refs, &refs, refs - 1)) { + if (clear_reading) + spin_unlock(&eb->refs_lock); return; + } } - spin_lock(&eb->refs_lock); + if (!clear_reading) + spin_lock(&eb->refs_lock); + if (refcount_read(&eb->refs) == 2 && test_bit(EXTENT_BUFFER_STALE, &eb->bflags) && !extent_buffer_under_io(eb) && @@ -3876,6 +4011,11 @@ void free_extent_buffer(struct extent_buffer *eb) release_extent_buffer(eb); } +void free_extent_buffer(struct extent_buffer *eb) +{ + return free_extent_buffer_clear_reading(eb, false); +} + void free_extent_buffer_stale(struct extent_buffer *eb) { if (!eb) @@ -4001,11 +4141,6 @@ void set_extent_buffer_uptodate(struct extent_buffer *eb) btrfs_meta_folio_set_uptodate(eb->folios[i], eb); } -static void clear_extent_buffer_reading(struct extent_buffer *eb) -{ - clear_and_wake_up_bit(EXTENT_BUFFER_READING, &eb->bflags); -} - static void end_bbio_meta_read(struct btrfs_bio *bbio) { struct extent_buffer *eb = bbio->private; @@ -4029,8 +4164,7 @@ static void end_bbio_meta_read(struct btrfs_bio *bbio) else clear_extent_buffer_uptodate(eb); - clear_extent_buffer_reading(eb); - free_extent_buffer(eb); + free_extent_buffer_clear_reading(eb, true); bio_put(&bbio->bio); } diff --git a/fs/btrfs/fiemap.c b/fs/btrfs/fiemap.c index 6263e837093e42..929e978702c352 100644 --- a/fs/btrfs/fiemap.c +++ b/fs/btrfs/fiemap.c @@ -660,7 +660,7 @@ restart: range_end = round_up(start + len, sectorsize); prev_extent_end = range_start; - btrfs_lock_extent(&inode->io_tree, range_start, range_end, &cached_state); + btrfs_lock_extent(&inode->io_tree, range_start, range_end - 1, &cached_state); ret = fiemap_find_last_extent_offset(inode, path, &last_extent_end); if (ret < 0) @@ -840,7 +840,7 @@ check_eof_delalloc: } out_unlock: - btrfs_unlock_extent(&inode->io_tree, range_start, range_end, &cached_state); + btrfs_unlock_extent(&inode->io_tree, range_start, range_end - 1, &cached_state); if (ret == BTRFS_FIEMAP_FLUSH_CACHE) { btrfs_release_path(path); diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index a2a2df2df78669..f949805f2c3ef3 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -875,62 +875,56 @@ again: /* * Locks the extent and properly waits for data=ordered extents to finish - * before allowing the folios to be modified if need. + * before allowing the folios to be modified. * * Return: - * 1 - the extent is locked - * 0 - the extent is not locked, and everything is OK + * 0 - the extent is locked * -EAGAIN - need to prepare the folios again */ static noinline int -lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct folio *folio, - loff_t pos, size_t write_bytes, - u64 *lockstart, u64 *lockend, bool nowait, - struct extent_state **cached_state) +lock_and_cleanup_extent(struct btrfs_inode *inode, struct folio *folio, + loff_t pos, size_t write_bytes, + u64 *lockstart, u64 *lockend, bool nowait, + struct extent_state **cached_state) { struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_ordered_extent *ordered; u64 start_pos; u64 last_pos; - int ret = 0; start_pos = round_down(pos, fs_info->sectorsize); last_pos = round_up(pos + write_bytes, fs_info->sectorsize) - 1; - if (start_pos < inode->vfs_inode.i_size) { - struct btrfs_ordered_extent *ordered; - - if (nowait) { - if (!btrfs_try_lock_extent(&inode->io_tree, start_pos, - last_pos, cached_state)) { - folio_unlock(folio); - folio_put(folio); - return -EAGAIN; - } - } else { - btrfs_lock_extent(&inode->io_tree, start_pos, last_pos, - cached_state); - } - - ordered = btrfs_lookup_ordered_range(inode, start_pos, - last_pos - start_pos + 1); - if (ordered && - ordered->file_offset + ordered->num_bytes > start_pos && - ordered->file_offset <= last_pos) { - btrfs_unlock_extent(&inode->io_tree, start_pos, last_pos, - cached_state); + if (nowait) { + if (!btrfs_try_lock_extent(&inode->io_tree, start_pos, + last_pos, cached_state)) { folio_unlock(folio); folio_put(folio); - btrfs_start_ordered_extent(ordered); - btrfs_put_ordered_extent(ordered); return -EAGAIN; } - if (ordered) - btrfs_put_ordered_extent(ordered); + } else { + btrfs_lock_extent(&inode->io_tree, start_pos, last_pos, + cached_state); + } - *lockstart = start_pos; - *lockend = last_pos; - ret = 1; + ordered = btrfs_lookup_ordered_range(inode, start_pos, + last_pos - start_pos + 1); + if (ordered && + ordered->file_offset + ordered->num_bytes > start_pos && + ordered->file_offset <= last_pos) { + btrfs_unlock_extent(&inode->io_tree, start_pos, last_pos, + cached_state); + folio_unlock(folio); + folio_put(folio); + btrfs_start_ordered_extent(ordered); + btrfs_put_ordered_extent(ordered); + return -EAGAIN; } + if (ordered) + btrfs_put_ordered_extent(ordered); + + *lockstart = start_pos; + *lockend = last_pos; /* * We should be called after prepare_one_folio() which should have locked @@ -938,7 +932,7 @@ lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct folio *folio, */ WARN_ON(!folio_test_locked(folio)); - return ret; + return 0; } /* @@ -1195,7 +1189,6 @@ static int copy_one_range(struct btrfs_inode *inode, struct iov_iter *iter, const u64 reserved_start = round_down(start, fs_info->sectorsize); u64 reserved_len; struct folio *folio = NULL; - int extents_locked; u64 lockstart; u64 lockend; bool only_release_metadata = false; @@ -1253,18 +1246,16 @@ again: reserved_len = last_block - reserved_start; } - extents_locked = lock_and_cleanup_extent_if_need(inode, folio, start, - write_bytes, &lockstart, - &lockend, nowait, - &cached_state); - if (extents_locked < 0) { - if (!nowait && extents_locked == -EAGAIN) + ret = lock_and_cleanup_extent(inode, folio, start, write_bytes, + &lockstart, &lockend, nowait, &cached_state); + if (ret < 0) { + if (!nowait) goto again; btrfs_delalloc_release_extents(inode, reserved_len); release_space(inode, *data_reserved, reserved_start, reserved_len, only_release_metadata); - return extents_locked; + return ret; } copied = copy_folio_from_iter_atomic(folio, offset_in_folio(folio, start), @@ -1288,11 +1279,8 @@ again: /* No copied bytes, unlock, release reserved space and exit. */ if (copied == 0) { - if (extents_locked) - btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, - &cached_state); - else - btrfs_free_extent_state(cached_state); + btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, + &cached_state); btrfs_delalloc_release_extents(inode, reserved_len); release_space(inode, *data_reserved, reserved_start, reserved_len, only_release_metadata); @@ -1311,17 +1299,7 @@ again: ret = btrfs_dirty_folio(inode, folio, start, copied, &cached_state, only_release_metadata); - /* - * If we have not locked the extent range, because the range's start - * offset is >= i_size, we might still have a non-NULL cached extent - * state, acquired while marking the extent range as delalloc through - * btrfs_dirty_page(). Therefore free any possible cached extent state - * to avoid a memory leak. - */ - if (extents_locked) - btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state); - else - btrfs_free_extent_state(cached_state); + btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state); btrfs_delalloc_release_extents(inode, reserved_len); if (ret) { diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index 502fb4a55cb227..f64a4537be315c 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -4339,12 +4339,13 @@ static int qgroup_free_reserved_data(struct btrfs_inode *inode, struct ulist_node *unode; struct ulist_iterator uiter; struct extent_changeset changeset; + const u32 sectorsize = root->fs_info->sectorsize; + const u64 aligned_start = round_down(start, sectorsize); + const u64 aligned_len = round_up(start + len, sectorsize) - aligned_start; u64 freed = 0; int ret; extent_changeset_init_bytes_only(&changeset); - len = round_up(start + len, root->fs_info->sectorsize); - start = round_down(start, root->fs_info->sectorsize); ULIST_ITER_INIT(&uiter); while ((unode = ulist_next(&reserved->range_changed, &uiter))) { @@ -4356,12 +4357,15 @@ static int qgroup_free_reserved_data(struct btrfs_inode *inode, extent_changeset_release(&changeset); - /* Only free range in range [start, start + len) */ - if (range_start >= start + len || - range_start + range_len <= start) + /* + * Only free the range within + * [aligned_start, aligned_start + aligned_len). + */ + if (range_start >= aligned_start + aligned_len || + range_start + range_len <= aligned_start) continue; - free_start = max(range_start, start); - free_len = min(start + len, range_start + range_len) - + free_start = max(range_start, aligned_start); + free_len = min(aligned_start + aligned_len, range_start + range_len) - free_start; /* * TODO: To also modify reserved->ranges_reserved to reflect diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 4eca2b5e4a9247..e4b259717c2516 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -1537,6 +1537,33 @@ static void clear_reloc_root(struct btrfs_root *root) clear_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state); } +/* Drop the reloc trees of a relocation that is being deferred and retried. */ +static void abort_reloc_roots(struct reloc_control *rc, struct list_head *list) +{ + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; + struct btrfs_root *reloc_root, *tmp; + + list_for_each_entry_safe(reloc_root, tmp, list, root_list) { + struct btrfs_root *root; + + root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, false); + if (!IS_ERR(root)) { + if (root->reloc_root == reloc_root) { + clear_reloc_root(root); + btrfs_put_root(reloc_root); + } + btrfs_put_root(root); + } + + btrfs_set_root_refs(&reloc_root->root_item, 0); + memset(&reloc_root->root_item.drop_progress, 0, sizeof(struct btrfs_disk_key)); + btrfs_set_root_drop_level(&reloc_root->root_item, 0); + + list_del_init(&reloc_root->root_list); + list_add_tail(&reloc_root->reloc_dirty_list, &rc->dirty_subvol_roots); + } +} + static int clean_dirty_subvols(struct reloc_control *rc) { struct btrfs_root *root; @@ -1876,8 +1903,7 @@ again: return err; } -static noinline_for_stack -void merge_reloc_roots(struct reloc_control *rc) +static noinline_for_stack int merge_reloc_roots(struct reloc_control *rc) { struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; struct btrfs_root *root; @@ -1975,7 +2001,15 @@ again: goto again; } out: - if (ret) { + if (btrfs_is_zoned(fs_info) && ret == -EAGAIN) { + abort_reloc_roots(rc, &reloc_roots); + + /* New reloc root may be added. */ + mutex_lock(&fs_info->reloc_mutex); + list_splice_init(&rc->reloc_roots, &reloc_roots); + mutex_unlock(&fs_info->reloc_mutex); + abort_reloc_roots(rc, &reloc_roots); + } else if (ret) { btrfs_handle_fs_error(fs_info, ret, NULL); free_reloc_roots(&reloc_roots); @@ -2001,6 +2035,7 @@ out: * * The remaining nodes will be cleaned up by put_reloc_control(). */ + return ret; } static void free_block_list(struct rb_root *blocks) @@ -3730,7 +3765,9 @@ restart: */ err = prepare_to_merge(rc, err); - merge_reloc_roots(rc); + ret = merge_reloc_roots(rc); + if (ret && !err) + err = ret; rc->merge_reloc_tree = false; unset_reloc_control(rc); @@ -5720,7 +5757,9 @@ int btrfs_recover_relocation(struct btrfs_fs_info *fs_info) goto out_drop_reloc_refs; release_recovered_fs_roots(&recovered_roots, false); - merge_reloc_roots(rc); + ret = merge_reloc_roots(rc); + if (ret) + goto out_unset; unset_reloc_control(rc); diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index 8f941972810088..80919590175fcd 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -698,8 +698,6 @@ again: goto alloc_fail; } - xa_init(&h->writeback_inhibited_ebs); - /* * If we are JOIN_NOLOCK we're already committing a transaction and * waiting on this guy, so we don't need to do the sb_start_intwrite @@ -2589,6 +2587,12 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) ret = btrfs_write_and_wait_transaction(trans); if (unlikely(ret)) { btrfs_err(fs_info, "error while writing out transaction: %d", ret); + /* + * Abort before releasing tree_log_mutex, so a log sync waiting + * on it sees the fs error and skips writing super_for_commit + * for this failed transaction. See btrfs_sync_log(). + */ + btrfs_abort_transaction(trans, ret); mutex_unlock(&fs_info->tree_log_mutex); goto scrub_continue; } diff --git a/fs/btrfs/transaction.h b/fs/btrfs/transaction.h index 5e4b1106fd9054..3a57f227b5ed28 100644 --- a/fs/btrfs/transaction.h +++ b/fs/btrfs/transaction.h @@ -7,12 +7,12 @@ #define BTRFS_TRANSACTION_H #include +#include #include #include #include #include #include -#include #include "btrfs_inode.h" #include "delayed-ref.h" @@ -23,6 +23,7 @@ struct btrfs_fs_info; struct btrfs_root_item; struct btrfs_root; struct btrfs_path; +struct extent_buffer; /* * Signal that a direct IO write is in progress, to avoid deadlock for sync @@ -136,6 +137,18 @@ enum { #define TRANS_EXTWRITERS (__TRANS_START | __TRANS_ATTACH) +/* + * Number of extent buffers a transaction handle tracks for writeback + * inhibition. The CLOCK reference bits pack into a u32 so this must not exceed + * 32, and keeping it a power of two lets the compiler reduce the CLOCK hand + * modulo to a mask. + */ +#define BTRFS_INHIBITED_EBS_SLOTS 8 + +static_assert(BTRFS_INHIBITED_EBS_SLOTS <= 32); +static_assert(BTRFS_INHIBITED_EBS_SLOTS != 0 && + (BTRFS_INHIBITED_EBS_SLOTS & (BTRFS_INHIBITED_EBS_SLOTS - 1)) == 0); + struct btrfs_trans_handle { u64 transid; u64 bytes_reserved; @@ -163,8 +176,14 @@ struct btrfs_trans_handle { struct btrfs_fs_info *fs_info; struct list_head new_bgs; struct btrfs_block_rsv delayed_rsv; - /* Extent buffers with writeback inhibited by this handle. */ - struct xarray writeback_inhibited_ebs; + + /* Extent buffers this handle has inhibited writeback on. */ + struct extent_buffer *inhibited_ebs[BTRFS_INHIBITED_EBS_SLOTS]; + /* CLOCK reference bit per slot. */ + u32 inhibited_ebs_referenced; + u32 nr_inhibited_ebs; + /* CLOCK hand. */ + u32 inhibited_ebs_hand; }; /* diff --git a/fs/btrfs/verity.c b/fs/btrfs/verity.c index 983365a7354168..1133a56c0568b9 100644 --- a/fs/btrfs/verity.c +++ b/fs/btrfs/verity.c @@ -720,14 +720,18 @@ again: goto out; folio_lock(folio); - /* If it's not uptodate after we have the lock, we got a read error. */ - if (!folio_test_uptodate(folio)) { + /* Folio was truncated from mapping. */ + if (!folio->mapping) { folio_unlock(folio); folio_put(folio); - return ERR_PTR(-EIO); + goto again; } - folio_unlock(folio); - goto out; + /* Another reader may have filled the folio while we waited. */ + if (folio_test_uptodate(folio)) { + folio_unlock(folio); + goto out; + } + goto read_folio; } folio = filemap_alloc_folio(mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS), @@ -744,6 +748,7 @@ again: return ERR_PTR(ret); } +read_folio: /* * Merkle item keys are indexed from byte 0 in the merkle tree. * They have the form: @@ -753,6 +758,7 @@ again: ret = read_key_bytes(BTRFS_I(inode), BTRFS_VERITY_MERKLE_ITEM_KEY, off, folio_address(folio), PAGE_SIZE, folio); if (ret < 0) { + folio_unlock(folio); folio_put(folio); return ERR_PTR(ret); } diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c index 40326517481cf8..12dd3dc5391528 100644 --- a/fs/ceph/dir.c +++ b/fs/ceph/dir.c @@ -1174,7 +1174,7 @@ static struct dentry *ceph_mkdir(struct mnt_idmap *idmap, struct inode *dir, !req->r_reply_info.head->is_target && !req->r_reply_info.head->is_dentry) err = ceph_handle_notrace_create(dir, dentry); - ret = ERR_PTR(err); + ret = err ? ERR_PTR(err) : NULL; out_req: if (!IS_ERR(ret) && req->r_dentry != dentry) /* Some other dentry was spliced in */ diff --git a/fs/ceph/file.c b/fs/ceph/file.c index a4a2a4b6a027d4..a0b9c2b5a58320 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -2477,6 +2477,54 @@ retry_snap: if (err < 0) goto out; + /* + * For O_APPEND writes we may have waited for Fwx exclusive caps + * while the previous Fwx holder (another client) extended the + * file. i_size has been updated via the cap grant message from + * the MDS, but ki_pos is still the old EOF. Re-read i_size here + * (no extra MDS round-trip needed) and adjust ki_pos to the true + * EOF. Since we hold Fwx, no other client can change the file. + */ + if (iocb->ki_flags & IOCB_APPEND) { + loff_t cur_eof = i_size_read(inode); + + if (cur_eof != pos) { + doutc(cl, + "%p %llx.%llx O_APPEND: pos adjusted %lld -> %lld\n", + inode, ceph_vinop(inode), pos, cur_eof); + iocb->ki_pos = cur_eof; + pos = cur_eof; + if (pos >= limit) { + err = -EFBIG; + goto out_caps; + } + iov_iter_truncate(from, limit - pos); + count = iov_iter_count(from); + + /* + * ceph_get_caps() validated the old endoff + * against i_max_size; adjusting ki_pos forward + * may have shifted the write range beyond the + * granted max_size. Re-check and truncate if + * necessary. + */ + spin_lock(&ci->i_ceph_lock); + if (pos + count > (loff_t)ci->i_max_size) { + loff_t max_size = ci->i_max_size; + + spin_unlock(&ci->i_ceph_lock); + if (pos >= max_size) { + err = -EFBIG; + goto out_caps; + } + iov_iter_truncate(from, max_size - pos); + count = iov_iter_count(from); + } else { + spin_unlock(&ci->i_ceph_lock); + } + } + } + err = file_update_time(file); if (err) goto out_caps; diff --git a/fs/dax.c b/fs/dax.c index 6d175cd47a99bc..df19c9317d10e8 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -392,12 +392,12 @@ int dax_folio_reset_order(struct folio *folio) int order = folio_order(folio); /* - * DAX maintains the invariant that folio->share != 0 only when - * folio->mapping == NULL (enforced by dax_folio_make_shared()). - * Equivalently: folio->mapping != NULL implies folio->share == 0. - * Callers ensure share has been decremented to zero before - * calling here, so unconditionally clearing both fields is - * correct. + * Clear the mapping and the index/share union word. folio->share + * and folio->index occupy the same union in struct folio. For + * non-shared folios (mapping != NULL), the union holds folio->index + * (file page offset); for shared folios (mapping == NULL), it holds + * folio->share (reference count). Either way, we are releasing the + * folio and both fields should be zeroed. */ folio->mapping = NULL; folio->share = 0; diff --git a/fs/dcache.c b/fs/dcache.c index 3e9af9de707464..073c2ce2d44486 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -1794,7 +1794,12 @@ static void do_one_tree(struct dentry *dentry) { shrink_dcache_tree(dentry, true); d_walk(dentry, dentry, umount_check); - d_drop(dentry); + spin_lock(&dentry->d_lock); + __d_drop(dentry); + /* A busy root survives the dput() below so don't leave it on ->s_roots. */ + if (unlikely(!hlist_unhashed(&dentry->d_sib))) + unlink_secondary_root(dentry); + spin_unlock(&dentry->d_lock); dput(dentry); } diff --git a/fs/erofs/Kconfig b/fs/erofs/Kconfig index 2dfc313588d283..37495e95a78d9b 100644 --- a/fs/erofs/Kconfig +++ b/fs/erofs/Kconfig @@ -132,11 +132,13 @@ config EROFS_FS_ZIP_LZMA Say N if you want to disable LZMA compression support. config EROFS_FS_ZIP_LZMA_DEFAULT_MAX_STREAMS - int "EROFS LZMA default maximum decompression streams" + int "EROFS LZMA default maximum decompression streams" if SMP depends on EROFS_FS_ZIP_LZMA range 1 NR_CPUS if SMP range 1 1 if !SMP - default 16 + default 16 if SMP && NR_CPUS >= 16 + default NR_CPUS if SMP + default 1 help By default EROFS allocates one LZMA decompression stream per CPU. Each stream can hold a dictionary of up to 8 MiB taken from the @@ -144,7 +146,7 @@ config EROFS_FS_ZIP_LZMA_DEFAULT_MAX_STREAMS of memory. This caps the default; the lzma_streams module parameter still overrides it. - If unsure, keep the default of 16. + If unsure, keep the suggested default (16 or lower). config EROFS_FS_ZIP_DEFLATE bool "EROFS DEFLATE compressed data support" diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c index 3c54e95964c9fb..27caf4bebddc37 100644 --- a/fs/erofs/decompressor.c +++ b/fs/erofs/decompressor.c @@ -299,6 +299,7 @@ static const char *z_erofs_transform_plain(struct z_erofs_decompress_req *rq, return ERR_PTR(-EOPNOTSUPP); if (rq->alg == Z_EROFS_COMPRESSION_INTERLACED) { cur = bs - (rq->pageofs_out & (bs - 1)); + DBG_BUGON(rq->pageofs_in & (bs - 1)); pi = (rq->pageofs_in + rq->inputsize - cur) & ~PAGE_MASK; cur = min(cur, rq->outputsize); if (cur && rq->out[0]) { diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h index 57bd21859c65d9..bee2f50c8488fa 100644 --- a/fs/erofs/internal.h +++ b/fs/erofs/internal.h @@ -269,7 +269,7 @@ struct erofs_inode { #ifdef CONFIG_EROFS_FS_ZIP struct { unsigned short z_advise; - unsigned char z_algorithmtype[2]; + unsigned char z_algofmt[2]; unsigned char z_lclusterbits; union { u64 z_tailextent_headlcn; diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index 74520e91025966..0e9cac8ee12d35 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -128,7 +128,17 @@ struct z_erofs_pcluster_slab { #define _PCLP(n) { .maxpages = n } static struct z_erofs_pcluster_slab pcluster_pool[] __read_mostly = { - _PCLP(1), _PCLP(4), _PCLP(16), _PCLP(64), _PCLP(128), + _PCLP(1), + _PCLP(4), +#if Z_EROFS_PCLUSTER_MAX_PAGES > 16 + _PCLP(16), +#endif +#if Z_EROFS_PCLUSTER_MAX_PAGES > 64 + _PCLP(64), +#endif +#if Z_EROFS_PCLUSTER_MAX_PAGES > 128 + _PCLP(128), +#endif _PCLP(Z_EROFS_PCLUSTER_MAX_PAGES + 1) }; diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c index 5811556a7b71fb..b316a50efe3839 100644 --- a/fs/erofs/zmap.c +++ b/fs/erofs/zmap.c @@ -483,14 +483,14 @@ static int z_erofs_map_blocks_fo(struct inode *inode, } if (m.headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN) { - if (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) + if ((vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) && + !(map->m_flags & EROFS_MAP_META)) map->m_algorithmformat = Z_EROFS_COMPRESSION_INTERLACED; else map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED; - } else if (m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) { - map->m_algorithmformat = vi->z_algorithmtype[1]; } else { - map->m_algorithmformat = vi->z_algorithmtype[0]; + map->m_algorithmformat = + vi->z_algofmt[m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2]; } if ((flags & EROFS_GET_BLOCKS_FIEMAP) || @@ -604,9 +604,14 @@ static int z_erofs_map_blocks_ext(struct inode *inode, if (map->m_plen & Z_EROFS_EXTENT_PLEN_PARTIAL) map->m_flags |= EROFS_MAP_PARTIAL_REF; map->m_plen &= Z_EROFS_EXTENT_PLEN_MASK; - if (fmt) - map->m_algorithmformat = fmt - 1; - else if (interlaced && !((map->m_pa | map->m_plen) & bmask)) + if (fmt) { + map->m_algorithmformat = --fmt; + if (fmt >= Z_EROFS_COMPRESSION_MAX) { + erofs_err(sb, "unknown algorithm %d @ pos %llu for nid %llu, please upgrade kernel", + fmt, map->m_la, vi->nid); + return -EOPNOTSUPP; + } + } else if (interlaced && !((map->m_pa | map->m_plen) & bmask)) map->m_algorithmformat = Z_EROFS_COMPRESSION_INTERLACED; else @@ -624,7 +629,7 @@ static int z_erofs_fill_inode(struct inode *inode, struct erofs_map_blocks *map) struct super_block *const sb = inode->i_sb; struct z_erofs_map_header *h; erofs_off_t pos; - int err = 0; + int err = 0, nr; if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) { /* @@ -667,12 +672,19 @@ static int z_erofs_fill_inode(struct inode *inode, struct erofs_map_blocks *map) goto done; } - vi->z_algorithmtype[0] = h->h_algorithmtype & 15; - vi->z_algorithmtype[1] = h->h_algorithmtype >> 4; if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff); else if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) vi->z_idata_size = le16_to_cpu(h->h_idata_size); + for (nr = 0; nr < 2; ++nr) { + vi->z_algofmt[nr] = (h->h_algorithmtype >> (4 * nr)) & 15; + if (vi->z_algofmt[nr] >= Z_EROFS_COMPRESSION_MAX) { + erofs_err(sb, "unknown HEAD%u format %u for nid %llu, please upgrade kernel", + nr + 1, vi->z_algofmt[nr], vi->nid); + err = -EOPNOTSUPP; + goto out_unlock; + } + } if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | @@ -720,12 +732,8 @@ static int z_erofs_map_sanity_check(struct inode *inode, if (!(map->m_flags & EROFS_MAP_MAPPED)) return 0; - if (unlikely(map->m_algorithmformat >= Z_EROFS_COMPRESSION_RUNTIME_MAX)) { - erofs_err(inode->i_sb, "unknown algorithm %d @ pos %llu for nid %llu, please upgrade kernel", - map->m_algorithmformat, map->m_la, EROFS_I(inode)->nid); - return -EOPNOTSUPP; - } + DBG_BUGON(map->m_algorithmformat >= Z_EROFS_COMPRESSION_RUNTIME_MAX); if (map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX) { if (!(sbi->available_compr_algs & BIT(map->m_algorithmformat))) { erofs_err(inode->i_sb, "inconsistent algorithmtype %u for nid %llu", diff --git a/fs/exfat/exfat_fs.h b/fs/exfat/exfat_fs.h index 9be50949ce34f5..1f020b041a3d6e 100644 --- a/fs/exfat/exfat_fs.h +++ b/fs/exfat/exfat_fs.h @@ -294,7 +294,7 @@ struct exfat_inode_info { /* on-disk position of directory entry or 0 */ loff_t i_pos; loff_t valid_size; - /* page-aligned size that has been zeroed out for mmap */ + /* block-aligned size zeroed in the page cache (>= valid_size) */ loff_t zeroed_size; /* hash by i_location */ struct hlist_node i_hash_fat; diff --git a/fs/exfat/file.c b/fs/exfat/file.c index 5fc13378d35f72..5e9b47ecc614e3 100644 --- a/fs/exfat/file.c +++ b/fs/exfat/file.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "exfat_raw.h" #include "exfat_fs.h" @@ -654,6 +655,104 @@ int exfat_file_fsync(struct file *filp, loff_t start, loff_t end, int datasync) return blkdev_issue_flush(inode->i_sb->s_bdev); } +/* + * exfat_zero_new_range - zero [start, end) without overwriting uptodate blocks + * + * Uptodate blocks may contain data written through a shared mapping beyond + * valid_size. + */ +static int exfat_zero_new_range(struct inode *inode, loff_t start, loff_t end) +{ + struct address_space *mapping = inode->i_mapping; + unsigned int blocksize = i_blocksize(inode); + loff_t pos = start; + int err; + + while (pos < end) { + loff_t next = min_t(loff_t, + round_down(pos, PAGE_SIZE) + PAGE_SIZE, end); + struct folio *folio; + loff_t bpos; + + folio = filemap_get_folio(mapping, pos >> PAGE_SHIFT); + if (IS_ERR(folio)) { + err = iomap_zero_range(inode, pos, next - pos, NULL, + &exfat_iomap_ops, NULL, NULL); + if (err < 0) + return err; + pos = next; + continue; + } + + if (folio_test_uptodate(folio)) { + folio_lock(folio); + if (folio->mapping == mapping) + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + pos = next; + continue; + } + + /* + * Zero not-uptodate block runs. iomap_zero_range() requires an + * unlocked folio, so recheck ->mapping after each call. + */ + folio_lock(folio); + bpos = pos; + while (bpos < next) { + loff_t rstart, rend; + + if (folio->mapping != mapping) { + folio_unlock(folio); + err = iomap_zero_range(inode, bpos, next - bpos, + NULL, &exfat_iomap_ops, NULL, NULL); + if (err < 0) { + folio_put(folio); + return err; + } + folio_lock(folio); + break; + } + + if (iomap_is_partially_uptodate(folio, + offset_in_folio(folio, bpos), blocksize)) { + bpos += blocksize; + continue; + } + + rstart = bpos; + rend = min_t(loff_t, bpos + blocksize, next); + while (rend < next && + !iomap_is_partially_uptodate(folio, + offset_in_folio(folio, rend), blocksize)) + rend = min_t(loff_t, rend + blocksize, next); + + folio_unlock(folio); + err = iomap_zero_range(inode, rstart, rend - rstart, + NULL, &exfat_iomap_ops, NULL, NULL); + if (err < 0) { + folio_put(folio); + return err; + } + folio_lock(folio); + bpos = rend; + } + + /* + * Dirty only a fully uptodate folio. Dirtying a partial folio could + * write uninitialised cache contents over valid on-disk blocks. + */ + if (folio->mapping == mapping && folio_test_uptodate(folio)) + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + pos = next; + } + + return 0; +} + static int exfat_extend_valid_size(struct inode *inode, loff_t new_valid_size) { struct exfat_inode_info *ei = EXFAT_I(inode); @@ -661,18 +760,41 @@ static int exfat_extend_valid_size(struct inode *inode, loff_t new_valid_size) int ret = 0; if (old_valid_size < new_valid_size) { + /* Do not re-zero blocks already covered by zeroed_size. */ + loff_t gap_start = max(old_valid_size, ei->zeroed_size); + if (i_size_read(inode) < new_valid_size) { - i_size_write(inode, new_valid_size); - mark_inode_dirty(inode); + /* + * Allocate clusters before increasing i_size. The gap + * may already be zeroed, so the subsequent zeroing + * can be skipped. + */ + ret = exfat_cont_expand(inode, new_valid_size); + if (ret) + return ret; } - ret = iomap_zero_range(inode, old_valid_size, - new_valid_size - old_valid_size, NULL, - &exfat_write_iomap_ops, NULL, NULL); + /* + * Revoke writable PTEs while zeroing the gap. A racing mmap + * store re-faults through exfat_page_mkwrite() after valid_size + * is updated. + */ + filemap_invalidate_lock(inode->i_mapping); + if (gap_start < new_valid_size) + unmap_mapping_range(inode->i_mapping, gap_start, + new_valid_size - gap_start, 0); + ret = exfat_zero_new_range(inode, gap_start, new_valid_size); + filemap_invalidate_unlock(inode->i_mapping); if (ret) { truncate_setsize(inode, old_valid_size); exfat_truncate(inode); + return ret; } + + ei->valid_size = new_valid_size; + if (ei->zeroed_size < round_up(new_valid_size, i_blocksize(inode))) + ei->zeroed_size = round_up(new_valid_size, i_blocksize(inode)); + mark_inode_dirty(inode); } return ret; @@ -825,39 +947,39 @@ static vm_fault_t exfat_page_mkwrite(struct vm_fault *vmf) struct inode *inode = file_inode(vmf->vma->vm_file); struct exfat_inode_info *ei = EXFAT_I(inode); vm_fault_t ret; - loff_t new_valid_size, mmap_valid_size; + loff_t new_valid_size, mmap_valid_size, fault_page_start; if (!inode_trylock(inode)) return VM_FAULT_RETRY; mmap_valid_size = ((loff_t)vmf->pgoff + 1) << PAGE_SHIFT; + fault_page_start = ((loff_t)vmf->pgoff) << PAGE_SHIFT; new_valid_size = min(mmap_valid_size, i_size_read(inode)); if (ei->valid_size < new_valid_size) { - if (ei->zeroed_size < mmap_valid_size) { + if (ei->zeroed_size < fault_page_start) { int err; /* - * Only zero the range that hasn't been zeroed yet for - * this mmap write path. zeroed_size tracks the largest - * page-aligned offset that has already been zeroed. - * - * This prevents unnecessarily zeroing out the entire - * tail page on every page fault when userspace writes - * data byte-by-byte through mmap (after a small - * fallocate). It fixes data corruption in the tail page - * while preserving the existing valid_size semantics. + * Zero only the gap below the faulting page. The read + * fault populated its folio and iomap_page_mkwrite() + * will dirty it. */ - err = iomap_zero_range(inode, ei->zeroed_size, - mmap_valid_size - ei->zeroed_size, NULL, - &exfat_iomap_ops, NULL, NULL); + err = exfat_zero_new_range(inode, ei->zeroed_size, + fault_page_start); if (err < 0) { inode_unlock(inode); return vmf_fs_error(err); } - ei->zeroed_size = mmap_valid_size; } + /* + * Track zeroed_size by block, not page, because writeback stops + * at i_size recording blocks wholly beyond it could skip a + * later required zeroing. + */ + if (ei->zeroed_size < round_up(new_valid_size, i_blocksize(inode))) + ei->zeroed_size = round_up(new_valid_size, i_blocksize(inode)); ei->valid_size = new_valid_size; mark_inode_dirty(inode); } @@ -866,7 +988,7 @@ static vm_fault_t exfat_page_mkwrite(struct vm_fault *vmf) file_update_time(vmf->vma->vm_file); filemap_invalidate_lock_shared(inode->i_mapping); - ret = iomap_page_mkwrite(vmf, &exfat_write_iomap_ops, NULL); + ret = iomap_page_mkwrite(vmf, &exfat_iomap_ops, NULL); filemap_invalidate_unlock_shared(inode->i_mapping); sb_end_pagefault(inode->i_sb); inode_unlock(inode); @@ -876,7 +998,6 @@ static vm_fault_t exfat_page_mkwrite(struct vm_fault *vmf) static const struct vm_operations_struct exfat_file_vm_ops = { .fault = filemap_fault, - .map_pages = filemap_map_pages, .page_mkwrite = exfat_page_mkwrite, }; @@ -887,21 +1008,6 @@ static int exfat_file_mmap_prepare(struct vm_area_desc *desc) if (unlikely(exfat_forced_shutdown(file_inode(desc->file)->i_sb))) return -EIO; - if (vma_desc_test_all(desc, VMA_SHARED_BIT, VMA_MAYWRITE_BIT)) { - struct inode *inode = file_inode(file); - loff_t from, to; - int err; - - from = ((loff_t)desc->pgoff << PAGE_SHIFT); - to = min_t(loff_t, i_size_read(inode), - from + vma_desc_size(desc)); - if (EXFAT_I(inode)->valid_size < to) { - err = exfat_extend_valid_size(inode, to); - if (err) - return err; - } - } - file_accessed(file); desc->vm_ops = &exfat_file_vm_ops; return 0; diff --git a/fs/exfat/iomap.c b/fs/exfat/iomap.c index 190fc6471f8493..d4d3ed933a63de 100644 --- a/fs/exfat/iomap.c +++ b/fs/exfat/iomap.c @@ -175,11 +175,18 @@ static int exfat_write_iomap_end(struct inode *inode, loff_t pos, loff_t length, if (ei->valid_size < end) { ei->valid_size = end; - if (ei->zeroed_size < end) - ei->zeroed_size = end; dirtied = true; } + /* + * IOMAP_F_ZERO_TAIL zeroes the remainder of the last block. Track that + * block as zeroed so later valid_size extensions do not zero it again. + */ + if (iomap->flags & IOMAP_F_ZERO_TAIL) + end = round_up(end, i_blocksize(inode)); + if (ei->zeroed_size < end) + ei->zeroed_size = end; + if (dirtied || iomap->flags & IOMAP_F_SIZE_CHANGED) mark_inode_dirty(inode); diff --git a/fs/ext4/dir.c b/fs/ext4/dir.c index 17edd678fa87b8..8d7b81e6948e72 100644 --- a/fs/ext4/dir.c +++ b/fs/ext4/dir.c @@ -138,6 +138,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) struct buffer_head *bh = NULL; struct fscrypt_str fstr = FSTR_INIT(NULL, 0); struct dir_private_info *info = file->private_data; + bool has_csum = ext4_has_feature_metadata_csum(sb); err = fscrypt_prepare_readdir(inode); if (err) @@ -149,7 +150,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) return err; /* Can we just clear INDEX flag to ignore htree information? */ - if (!ext4_has_feature_metadata_csum(sb)) { + if (!has_csum) { /* * We don't set the inode dirty flag since it's not * critical that it gets flushed back to the disk. @@ -235,7 +236,10 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) * dirent right now. Scan from the start of the block * to make sure. */ if (!inode_eq_iversion(inode, info->cookie)) { - for (i = 0; i < sb->s_blocksize && i < offset; ) { + for (i = 0; + i <= sb->s_blocksize - + ext4_dir_rec_len(1, has_csum ? NULL : inode) && + i < offset;) { de = (struct ext4_dir_entry_2 *) (bh->b_data + i); /* It's too expensive to do a full @@ -257,6 +261,17 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) info->cookie = inode_query_iversion(inode); } + if (unlikely(offset < sb->s_blocksize && + offset > sb->s_blocksize - + ext4_dir_rec_len(1, has_csum ? NULL : inode))) { + EXT4_ERROR_FILE(file, bh->b_blocknr, + "bad entry in directory: %s - offset=%u, size=%lu", + "directory entry too close to block end", + offset, sb->s_blocksize); + ctx->pos = round_up(ctx->pos, sb->s_blocksize); + goto next_block; + } + while (ctx->pos < inode->i_size && offset < sb->s_blocksize) { de = (struct ext4_dir_entry_2 *) (bh->b_data + offset); @@ -312,6 +327,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) ctx->pos += ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize); } +next_block: if ((ctx->pos < inode->i_size) && !dir_relax_shared(inode)) goto done; brelse(bh); diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index b37c136ea3ab33..ed4241797ae34c 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -3138,13 +3138,15 @@ int do_journal_get_write_access(handle_t *handle, struct inode *inode, void ext4_set_inode_mapping_order(struct inode *inode); #define FALL_BACK_TO_NONDELALLOC 1 #define CONVERT_INLINE_DATA 2 +#define EXT4_WRITE_DATA_INLINE 4 typedef enum { EXT4_IGET_NORMAL = 0, EXT4_IGET_SPECIAL = 0x0001, /* OK to iget a system inode */ EXT4_IGET_HANDLE = 0x0002, /* Inode # is from a handle */ EXT4_IGET_BAD = 0x0004, /* Allow to iget a bad inode */ - EXT4_IGET_EA_INODE = 0x0008 /* Inode should contain an EA value */ + EXT4_IGET_EA_INODE = 0x0008, /* Inode should contain an EA value */ + EXT4_IGET_NOWAIT = 0x0010 /* Non-blocking lookup (skip if freeing) */ } ext4_iget_flags; extern struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, @@ -3184,7 +3186,7 @@ extern int ext4_normal_submit_inode_data_buffers(struct jbd2_inode *jinode); extern int ext4_chunk_trans_blocks(struct inode *, int nrblocks); extern int ext4_chunk_trans_extent(struct inode *inode, int nrblocks); extern int ext4_meta_trans_blocks(struct inode *inode, int lblocks, - int pextents); + int pextents, int alloc_extents); extern int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end); extern int ext4_zero_partial_blocks(struct inode *inode, loff_t lstart, loff_t length, bool *did_zero); diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index 91c97af64b3177..44ab246a317628 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -4976,7 +4976,7 @@ int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode, * it can tell if the extent in the cache is a split extent. * But for now let's assume pextents as 2 always. */ - credits = ext4_meta_trans_blocks(inode, max_blocks, 2); + credits = ext4_meta_trans_blocks(inode, max_blocks, 2, 0); } if (credits) { diff --git a/fs/ext4/file.c b/fs/ext4/file.c index eb1a323962b101..130edf1ac2423f 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -309,6 +309,13 @@ static ssize_t ext4_buffered_write_iter(struct kiocb *iocb, return -EOPNOTSUPP; inode_lock(inode); + + /* + * Prevent concurrent direct I/O and buffered I/O to the same file + * range. Wait for in-flight DIO to finish before dirtying pages. + */ + inode_dio_wait(inode); + ret = ext4_write_checks(iocb, from); if (ret <= 0) goto out; diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c index 8045e4ff270c7f..7bb28735de9110 100644 --- a/fs/ext4/inline.c +++ b/fs/ext4/inline.c @@ -812,7 +812,19 @@ int ext4_write_inline_data_end(struct inode *inode, loff_t pos, unsigned len, goto out; } ext4_write_lock_xattr(inode, &no_expand); - BUG_ON(!ext4_has_inline_data(inode)); + /* + * We could have raced with ext4_page_mkwrite() converting + * the inode and clearing the inline data flag, so we just + * release resources and retry the whole write. + */ + if (unlikely(!ext4_has_inline_data(inode))) { + ext4_write_unlock_xattr(inode, &no_expand); + brelse(iloc.bh); + folio_unlock(folio); + folio_put(folio); + ext4_journal_stop(handle); + return 0; + } /* * ei->i_inline_off may have changed since @@ -1454,6 +1466,8 @@ int ext4_read_inline_dir(struct file *file, /* for other entry, the real offset in * the buf has to be tuned accordingly. */ + if (i + ext4_dir_rec_len(1, NULL) > extra_size) + break; de = (struct ext4_dir_entry_2 *) (dir_buf + i - extra_offset); /* It's too expensive to do a full @@ -1488,10 +1502,17 @@ int ext4_read_inline_dir(struct file *file, continue; } + /* + * de lives at dir_buf + ctx->pos - extra_offset, within the + * kmalloc(inline_size) buffer. Make sure its header fits before + * ext4_check_dir_entry() dereferences de->rec_len. + */ + if (ctx->pos + ext4_dir_rec_len(1, NULL) > extra_size) + goto out; de = (struct ext4_dir_entry_2 *) (dir_buf + ctx->pos - extra_offset); if (ext4_check_dir_entry(inode, file, de, iloc.bh, dir_buf, - extra_size, ctx->pos)) + inline_size, ctx->pos)) goto out; if (le32_to_cpu(de->inode)) { if (!dir_emit(ctx, de->name, de->name_len, diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index ed39c71504bf40..a33629b397b060 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -1303,6 +1303,8 @@ static int ext4_write_begin(const struct kiocb *iocb, if (unlikely(ret)) return ret; + *fsdata = (void *)((unsigned long)*fsdata & ~EXT4_WRITE_DATA_INLINE); + trace_ext4_write_begin(inode, pos, len); /* * Reserve one block more for addition to orphan list in case @@ -1317,8 +1319,10 @@ static int ext4_write_begin(const struct kiocb *iocb, foliop); if (ret < 0) return ret; - if (ret == 1) + if (ret == 1) { + *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE); return 0; + } } /* @@ -1451,8 +1455,7 @@ static int ext4_write_end(const struct kiocb *iocb, trace_ext4_write_end(inode, pos, len, copied); - if (ext4_has_inline_data(inode) && - ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) + if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -1561,8 +1564,7 @@ static int ext4_journalled_write_end(const struct kiocb *iocb, BUG_ON(!ext4_handle_valid(handle)); - if (ext4_has_inline_data(inode) && - ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) + if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -2695,13 +2697,25 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) * page is already under writeback and we are not doing * a data integrity writeback, skip the page */ - if (!folio_test_dirty(folio) || - (folio_test_writeback(folio) && - (mpd->wbc->sync_mode == WB_SYNC_NONE)) || + if ((folio_test_writeback(folio) && + mpd->wbc->sync_mode == WB_SYNC_NONE) || unlikely(folio->mapping != mapping)) { folio_unlock(folio); continue; } + /* + * If the folio is clean, skip writing it back. + * Cycle the folio through the writeback state + * though, to clear stale xarray tags. + */ + if (!folio_test_dirty(folio)) { + if (!folio_test_writeback(folio)) { + __folio_start_writeback(folio, false); + folio_end_writeback(folio); + } + folio_unlock(folio); + continue; + } folio_wait_writeback(folio); BUG_ON(folio_test_writeback(folio)); @@ -2853,10 +2867,10 @@ static int ext4_do_writepages(struct mpage_da_data *mpd) if (ext4_should_dioread_nolock(inode)) { int bpf = ext4_journal_blocks_per_folio(inode); /* - * We may need to convert up to one extent per block in - * the folio and we may dirty the inode. + * We may need to convert up to one extent per block in the + * folio. */ - rsv_blocks = 1 + ext4_ext_index_trans_blocks(inode, bpf); + rsv_blocks = ext4_meta_trans_blocks(inode, bpf, bpf, 0); } if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) @@ -3162,8 +3176,10 @@ static int ext4_da_write_begin(const struct kiocb *iocb, foliop, fsdata, true); if (ret < 0) return ret; - if (ret == 1) + if (ret == 1) { + *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE); return 0; + } } retry: @@ -3292,17 +3308,15 @@ static int ext4_da_write_end(const struct kiocb *iocb, struct folio *folio, void *fsdata) { struct inode *inode = mapping->host; - int write_mode = (int)(unsigned long)fsdata; + unsigned long write_mode = (unsigned long)fsdata; - if (write_mode == FALL_BACK_TO_NONDELALLOC) + if (write_mode & FALL_BACK_TO_NONDELALLOC) return ext4_write_end(iocb, mapping, pos, len, copied, folio, fsdata); trace_ext4_da_write_end(inode, pos, len, copied); - if (write_mode != CONVERT_INLINE_DATA && - ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) && - ext4_has_inline_data(inode)) + if (write_mode & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -3695,8 +3709,8 @@ static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map, return ret; if (map->m_len < orig_mlen) { map->m_len = orig_mlen; - dio_credits = ext4_meta_trans_blocks(inode, orig_mlen, - map->m_len); + dio_credits = ext4_meta_trans_blocks(inode, map->m_len, + map->m_len, 0); } else { dio_credits = ext4_chunk_trans_blocks(inode, map->m_len); @@ -4220,6 +4234,14 @@ int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end) offset = from & (blocksize - 1); if (!offset || from >= end) return 0; + /* + * Inline data has no tail block to zero out. Note that a race with + * ext4_page_mkwrite() converting inline data to an extent without + * holding i_rwsem is safe, as that path zeroes the full block before + * copying in the inline data. + */ + if (ext4_has_inline_data(inode)) + return 0; /* If we are processing an encrypted inode during orphan list handling */ if (IS_ENCRYPTED(inode) && !fscrypt_has_encryption_key(inode)) return 0; @@ -5271,6 +5293,20 @@ void ext4_set_inode_mapping_order(struct inode *inode) mapping_set_folio_order_range(inode->i_mapping, min_order, max_order); } +static int ext4_iget_match(struct inode *inode, u64 ino, void *data) +{ + if (inode->i_ino != ino) + return 0; + spin_lock(&inode->i_lock); + if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE | I_CREATING)) { + spin_unlock(&inode->i_lock); + return -1; + } + __iget(inode); + spin_unlock(&inode->i_lock); + return 1; +} + struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, ext4_iget_flags flags, const char *function, unsigned int line) @@ -5299,9 +5335,24 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, return ERR_PTR(-EFSCORRUPTED); } - inode = iget_locked(sb, ino); - if (!inode) - return ERR_PTR(-ENOMEM); + if (flags & EXT4_IGET_NOWAIT) { + inode = find_inode_nowait(sb, ino, ext4_iget_match, NULL); + if (!inode) + return ERR_PTR(-ENOENT); + + if (inode_state_read_once(inode) & I_NEW) + wait_on_new_inode(inode); + + if (unlikely(inode_unhashed(inode))) { + iput(inode); + return ERR_PTR(-ENOENT); + } + } else { + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + } + if (!(inode_state_read_once(inode) & I_NEW)) { ret = check_igot_inode(inode, flags, function, line); if (ret) { @@ -6308,17 +6359,17 @@ static int ext4_index_trans_blocks(struct inode *inode, int lblocks, } /* - * Account for index blocks, block groups bitmaps and block group - * descriptor blocks if modify datablocks and index blocks - * worse case, the indexs blocks spread over different block groups - * - * If datablocks are discontiguous, they are possible to spread over - * different block groups too. If they are contiguous, with flexbg, - * they could still across block group boundary. - * - * Also account for superblock, inode, quota and xattr blocks + * Calculate number of credits needed in a transaction to: + * * Allocate data blocks from @alloc_extents different groups - note that + * with flexbg a single physical extent can span multiple groups but + * single mballoc request only returns extent within one group. + * * Allocate metatadata (extent tree blocks, indirect blocks) to store + * pointers to @pextents data extents having @lblocks in total. + * * Modify extent tree / indirect block tree, inode, superblock, quota + * tracking, xattr blocks */ -int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents) +int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents, + int alloc_extents) { ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb); int gdpblocks; @@ -6335,7 +6386,7 @@ int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents) * Now let's see how many group bitmaps and group descriptors need * to account */ - groups = idxblocks + pextents; + groups = idxblocks + alloc_extents; gdpblocks = groups; if (groups > ngroups) groups = ngroups; @@ -6361,7 +6412,7 @@ int ext4_chunk_trans_extent(struct inode *inode, int nrblocks) { int ret; - ret = ext4_meta_trans_blocks(inode, nrblocks, 1); + ret = ext4_meta_trans_blocks(inode, nrblocks, 1, 1); /* Account for data blocks for journalled mode */ if (ext4_should_journal_data(inode)) ret += nrblocks; @@ -6379,7 +6430,7 @@ int ext4_chunk_trans_extent(struct inode *inode, int nrblocks) */ int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks) { - return ext4_meta_trans_blocks(inode, nrblocks, 1); + return ext4_meta_trans_blocks(inode, nrblocks, 1, 1); } /* @@ -6511,6 +6562,16 @@ static int ext4_try_to_expand_extra_isize(struct inode *inode, if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) return -EOVERFLOW; + /* + * Skip expansion during mount (!SB_ACTIVE). Expanding extra isize + * may move xattrs to external blocks and release ea_inodes via iput. + * When !SB_ACTIVE, iput triggers write_inode_now() which acquires + * s_writepages_rwsem, causing a deadlock with the caller's active + * jbd2 handle (lock order: s_writepages_rwsem -> jbd2_handle). + */ + if (unlikely(!(inode->i_sb->s_flags & SB_ACTIVE))) + return -EBUSY; + /* * In nojournal mode, we can immediately attempt to expand * the inode. When journaled, we first need to obtain extra diff --git a/fs/ext4/migrate.c b/fs/ext4/migrate.c index 477d43d7e2942b..5d60ef10fe1100 100644 --- a/fs/ext4/migrate.c +++ b/fs/ext4/migrate.c @@ -464,6 +464,7 @@ int ext4_ext_migrate(struct inode *inode) if (IS_ERR(tmp_inode)) { retval = PTR_ERR(tmp_inode); ext4_journal_stop(handle); + tmp_inode = NULL; goto out_unlock; } /* @@ -591,9 +592,9 @@ out_stop: ext4_journal_stop(handle); out_tmp_inode: unlock_new_inode(tmp_inode); - iput(tmp_inode); out_unlock: ext4_writepages_up_write(inode->i_sb, alloc_ctx); + iput(tmp_inode); return retval; } diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index cc49ae04a6f649..3b9740c1c16d8c 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -1467,6 +1467,8 @@ int ext4_search_dir(struct buffer_head *bh, char *search_buf, int buf_size, /* this code is executed quadratically often */ /* do minimal checking `by hand' */ if (de->name + de->name_len <= dlimit && + (!ext4_hash_in_dirent(dir) || + (char *)de + ext4_dir_rec_len(de->name_len, dir) <= dlimit) && ext4_match(dir, fname, de)) { /* found a match - just to be sure, do * a full check */ diff --git a/fs/ext4/orphan.c b/fs/ext4/orphan.c index 64ea4762423386..e7fe1a99d4855b 100644 --- a/fs/ext4/orphan.c +++ b/fs/ext4/orphan.c @@ -388,7 +388,7 @@ void ext4_orphan_cleanup(struct super_block *sb, struct ext4_super_block *es) struct ext4_orphan_info *oi = &EXT4_SB(sb)->s_orphan_info; int inodes_per_ob = ext4_inodes_per_orphan_block(sb); - if (!es->s_last_orphan && !oi->of_blocks) { + if (!es->s_last_orphan && ext4_orphan_file_empty(sb)) { ext4_debug("no orphan inodes to clean up\n"); return; } @@ -572,6 +572,7 @@ int ext4_init_orphan_info(struct super_block *sb) int i, j; int ret; int free; + int loaded = 0; __le32 *bdata; int inodes_per_ob = ext4_inodes_per_orphan_block(sb); struct ext4_orphan_block_tail *ot; @@ -613,6 +614,7 @@ int ext4_init_orphan_info(struct super_block *sb) ret = -EIO; goto out_free; } + loaded++; ot = ext4_orphan_block_tail(sb, oi->of_binfo[i].ob_bh); if (le32_to_cpu(ot->ob_magic) != EXT4_ORPHAN_BLOCK_MAGIC) { ext4_error(sb, "orphan file block %d: bad magic", i); @@ -635,8 +637,10 @@ int ext4_init_orphan_info(struct super_block *sb) iput(inode); return 0; out_free: - for (i--; i >= 0; i--) - brelse(oi->of_binfo[i].ob_bh); + while (loaded > 0) { + loaded--; + brelse(oi->of_binfo[loaded].ob_bh); + } kvfree(oi->of_binfo); out_put: iput(inode); diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c index 77512e709543e6..6fa41c48f39719 100644 --- a/fs/ext4/xattr.c +++ b/fs/ext4/xattr.c @@ -1550,7 +1550,7 @@ ext4_xattr_inode_cache_find(struct inode *inode, const void *value, while (ce) { ea_inode = ext4_iget(inode->i_sb, ce->e_value, - EXT4_IGET_EA_INODE); + EXT4_IGET_EA_INODE | EXT4_IGET_NOWAIT); if (IS_ERR(ea_inode)) goto next_entry; ext4_xattr_inode_set_class(ea_inode); diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 8011bbdf2c6888..bc2246abd421b4 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -4123,10 +4123,26 @@ static inline struct inode *fio_inode(struct f2fs_io_info *fio) #define MIN_FRAGMENT_SIZE 1 #define MAX_FRAGMENT_SIZE 512 -static inline bool f2fs_need_rand_seg(struct f2fs_sb_info *sbi) +static inline bool f2fs_need_rand_blk(struct f2fs_sb_info *sbi, + enum log_type type) { - return F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_SEG || - F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK; + if (type == CURSEG_COLD_DATA_PINNED) + return false; + return F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK; +} + +static inline bool f2fs_need_rand_seg(struct f2fs_sb_info *sbi, + enum log_type type) +{ + if (type == CURSEG_COLD_DATA_PINNED) + return false; + return F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_SEG; +} + +static inline bool f2fs_need_rand_seg_blk(struct f2fs_sb_info *sbi, + enum log_type type) +{ + return f2fs_need_rand_blk(sbi, type) || f2fs_need_rand_seg(sbi, type); } /* diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 67fcea7408b1be..f5c169dced2ddd 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -5379,9 +5379,8 @@ static ssize_t f2fs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) bool dio; bool may_need_sync = true; int preallocated; - const loff_t pos = iocb->ki_pos; - const ssize_t count = iov_iter_count(from); ssize_t ret; + loff_t bufio_start_pos; if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) { ret = -EIO; @@ -5402,15 +5401,17 @@ static ssize_t f2fs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) inode_lock(inode); } + ret = f2fs_write_checks(iocb, from); + if (ret <= 0) + goto out_unlock; + if (f2fs_is_pinned_file(inode) && - !f2fs_overwrite_io(inode, pos, count)) { + !f2fs_overwrite_io(inode, iocb->ki_pos, iov_iter_count(from))) { ret = -EIO; goto out_unlock; } - ret = f2fs_write_checks(iocb, from); - if (ret <= 0) - goto out_unlock; + bufio_start_pos = iocb->ki_pos; /* Determine whether we will do a direct write or a buffered write. */ dio = f2fs_should_use_dio(inode, iocb, from); @@ -5465,8 +5466,8 @@ out: */ if (ret > 0 && !dio && (iocb->ki_flags & IOCB_DIRECT)) f2fs_flush_buffered_write(iocb->ki_filp->f_mapping, - orig_pos, - orig_pos + ret - 1); + bufio_start_pos, + bufio_start_pos + ret - 1); return ret; } diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index 1eda8186197471..c771adedff39c9 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -310,7 +310,7 @@ static void select_policy(struct f2fs_sb_info *sbi, int gc_type, p->max_search = sbi->max_victim_search; /* let's select beginning hot/small space first. */ - if (f2fs_need_rand_seg(sbi)) { + if (f2fs_need_rand_seg_blk(sbi, type)) { p->offset = get_random_u32_below(MAIN_SECS(sbi) * SEGS_PER_SEC(sbi)); SIT_I(sbi)->last_victim[p->gc_mode] = p->offset; diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 091bfb170d615f..8830e1892b4353 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -2936,7 +2936,7 @@ static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) sanity_check_seg_type(sbi, seg_type); if (__is_large_section(sbi)) { - if (f2fs_need_rand_seg(sbi)) { + if (f2fs_need_rand_seg_blk(sbi, type)) { unsigned int hint = GET_SEC_FROM_SEG(sbi, curseg->segno); if (GET_SEC_FROM_SEG(sbi, curseg->segno + 1) != hint) @@ -2945,7 +2945,7 @@ static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) GET_SEG_FROM_SEC(sbi, hint + 1) - 1); } return curseg->segno; - } else if (f2fs_need_rand_seg(sbi)) { + } else if (f2fs_need_rand_seg_blk(sbi, type)) { return get_random_u32_below(MAIN_SECS(sbi) * SEGS_PER_SEC(sbi)); } @@ -3001,7 +3001,7 @@ static int new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) curseg->next_segno = segno; reset_curseg(sbi, type, 1); curseg->alloc_type = LFS; - if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) + if (f2fs_need_rand_blk(sbi, type)) curseg->fragment_remained_chunk = get_random_u32_inclusive(1, sbi->max_fragment_chunk); return 0; @@ -3822,7 +3822,7 @@ int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio, curseg->next_blkoff = f2fs_find_next_ssr_block(sbi, curseg); } else { curseg->next_blkoff++; - if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) + if (f2fs_need_rand_blk(sbi, type)) f2fs_randomize_chunk(sbi, curseg); } if (curseg->next_blkoff >= f2fs_usable_blks_in_seg(sbi, curseg->segno)) diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index f7e2dcb58b4db3..7f6c4e9d3735a2 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -2456,9 +2456,9 @@ static int f2fs_show_options(struct seq_file *seq, struct dentry *root) seq_puts(seq, "adaptive"); else if (F2FS_OPTION(sbi).fs_mode == FS_MODE_LFS) seq_puts(seq, "lfs"); - else if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_SEG) + else if (f2fs_need_rand_seg(sbi, NO_CHECK_TYPE)) seq_puts(seq, "fragment:segment"); - else if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) + else if (f2fs_need_rand_blk(sbi, NO_CHECK_TYPE)) seq_puts(seq, "fragment:block"); seq_printf(seq, ",active_logs=%u", F2FS_OPTION(sbi).active_logs); if (test_opt(sbi, RESERVE_ROOT) || test_opt(sbi, RESERVE_NODE)) diff --git a/fs/fat/nfs.c b/fs/fat/nfs.c index 509eea96a457d4..6e1b371711edd5 100644 --- a/fs/fat/nfs.c +++ b/fs/fat/nfs.c @@ -250,8 +250,10 @@ struct inode *fat_rebuild_parent(struct super_block *sb, int parent_logstart) MSDOS_I(dummy_grand_parent)->i_pos = -1; } - if (!fat_scan_logstart(dummy_grand_parent, clus_to_match, &sinfo)) + if (!fat_scan_logstart(dummy_grand_parent, clus_to_match, &sinfo)) { parent = fat_build_inode(sb, sinfo.de, sinfo.i_pos); + brelse(sinfo.bh); + } brelse(parent_bh); iput(dummy_grand_parent); diff --git a/fs/fuse/cuse.c b/fs/fuse/cuse.c index 3c15b5ba16d71e..4079cf8e5974ad 100644 --- a/fs/fuse/cuse.c +++ b/fs/fuse/cuse.c @@ -530,7 +530,8 @@ static int cuse_channel_open(struct inode *inode, struct file *file) INIT_LIST_HEAD(&cc->list); - cc->fc.chan->initialized = 1; + /* Pairs with smp_load_acquire() readers of fch->initialized */ + smp_store_release(&cc->fc.chan->initialized, 1); rc = cuse_send_init(cc); if (rc) { fuse_dev_put(fud); @@ -653,6 +654,11 @@ static void __exit cuse_exit(void) { misc_deregister(&cuse_miscdev); class_destroy(cuse_class); + /* + * Wait for pending call_rcu() callbacks that call back into + * this module via fc->release (cuse_fc_release). + */ + rcu_barrier(); } module_init(cuse_init); diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index e7da62a7293dcb..a82cf763f50db8 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -80,20 +80,17 @@ void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *pa fuse_uring_conn_init(fch); } - /* Make sure stores before this are seen on another CPU */ - smp_wmb(); - fch->initialized = 1; + /* Pairs with smp_load_acquire() readers of fch->initialized */ + smp_store_release(&fch->initialized, 1); wake_up_all(&fch->blocked_waitq); } static bool fuse_block_alloc(struct fuse_chan *fch, bool for_background) { - if (!fch->initialized) + /* Pairs with smp_store_release() in fuse_chan_set_initialized() */ + if (!smp_load_acquire(&fch->initialized)) return true; - /* Pairs with smp_wmb() in fuse_chan_set_initialized() */ - smp_rmb(); - return (for_background && fch->blocked) || (fch->io_uring && fch->connected && !fuse_uring_ready(fch)); } @@ -1891,7 +1888,8 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud, * initialized and connected state */ err = -EINVAL; - if (!fch->initialized || !fch->connected) + /* Pairs with smp_store_release() in fuse_chan_set_initialized() */ + if (!smp_load_acquire(&fch->initialized) || !fch->connected) goto copy_finish; /* Don't try to move folios (yet) */ diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 82da15a9cc787a..8448fdbd90c9ac 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -1237,23 +1237,30 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) } fch = fud->chan; - /* Once a connection has io-uring enabled on it, it can't be disabled */ - if (!enable_uring && !fch->io_uring) { - pr_info_ratelimited("fuse-io-uring is disabled\n"); - return -EOPNOTSUPP; - } + /* + * The ring is sized from values negotiated by FUSE_INIT + * + * Pairs with smp_store_release() in fuse_chan_set_initialized() + */ + if (!smp_load_acquire(&fch->initialized)) + return -EAGAIN; if (fch->abort_with_err) return -ECONNABORTED; if (!fch->connected) return -ENOTCONN; - /* - * fuse_uring_register() needs the ring to be initialized, - * we need to know the max payload size - */ - if (!fch->initialized) - return -EAGAIN; + /* Once a connection has io-uring enabled on it, it can't be disabled */ + if (!enable_uring && !fch->io_uring) { + pr_info_ratelimited("fuse-io-uring is disabled by module parameter\n"); + return -EOPNOTSUPP; + } + + if (!fch->io_uring) { + pr_info_ratelimited( + "fuse-io-uring not enabled on this connection\n"); + return -EOPNOTSUPP; + } switch (cmd_op) { case FUSE_IO_URING_CMD_REGISTER: diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 48763bc192f3c4..4466db3e38274e 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -96,6 +96,7 @@ static void fuse_advise_use_readdirplus(struct inode *dir) struct fuse_dentry { u64 time; + u64 epoch; union { struct rcu_head rcu; struct rb_node node; @@ -236,6 +237,13 @@ void fuse_dentry_tree_cleanup(void) WARN_ON_ONCE(!RB_EMPTY_ROOT(&dentry_hash[i].tree)); } +void fuse_dentry_set_epoch(struct dentry *dentry, u64 epoch) +{ + struct fuse_dentry *fd = dentry->d_fsdata; + + fd->epoch = epoch; +} + static inline void __fuse_dentry_settime(struct dentry *dentry, u64 time) { ((struct fuse_dentry *) dentry->d_fsdata)->time = time; @@ -387,10 +395,11 @@ static int fuse_dentry_revalidate(struct inode *dir, const struct qstr *name, struct fuse_mount *fm; struct fuse_conn *fc; struct fuse_inode *fi; + struct fuse_dentry *fd = entry->d_fsdata; int ret; fc = get_fuse_conn_super(dir->i_sb); - if (entry->d_time < atomic_read(&fc->epoch)) + if (fd->epoch < atomic_read(&fc->epoch)) goto invalid; inode = d_inode_rcu(entry); @@ -480,10 +489,10 @@ static int fuse_dentry_init(struct dentry *dentry) RB_CLEAR_NODE(&fd->node); dentry->d_fsdata = fd; /* - * Initialising d_time (epoch) to '0' ensures the dentry is invalid + * Initialising epoch to '0' ensures the dentry is invalid * if compared to fc->epoch, which is initialized to '1'. */ - dentry->d_time = 0; + fuse_dentry_set_epoch(dentry, 0); return 0; } @@ -641,7 +650,7 @@ static struct dentry *fuse_lookup(struct inode *dir, struct dentry *entry, goto out_err; entry = newent ? newent : entry; - entry->d_time = epoch; + fuse_dentry_set_epoch(entry, epoch); if (outarg_valid) fuse_change_entry_timeout(entry, &outarg); else @@ -898,7 +907,7 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, } kfree(forget); d_instantiate(entry, inode); - entry->d_time = epoch; + fuse_dentry_set_epoch(entry, epoch); fuse_change_entry_timeout(entry, &outentry); fuse_dir_changed(dir); err = generic_file_open(inode, file); @@ -1028,10 +1037,10 @@ static struct dentry *create_new_entry(struct mnt_idmap *idmap, struct fuse_moun return d; if (d) { - d->d_time = epoch; + fuse_dentry_set_epoch(d, epoch); fuse_change_entry_timeout(d, &outarg); } else { - entry->d_time = epoch; + fuse_dentry_set_epoch(entry, epoch); fuse_change_entry_timeout(entry, &outarg); } fuse_dir_changed(dir); diff --git a/fs/fuse/file.c b/fs/fuse/file.c index e7b2a839f08144..5820def81d9f5e 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1801,13 +1801,14 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct inode *inode = file_inode(iocb->ki_filp); struct address_space *mapping = inode->i_mapping; - loff_t pos = iocb->ki_pos; ssize_t res; bool exclusive; fuse_dio_lock(iocb, from, &exclusive); res = generic_write_checks(iocb, from); if (res > 0) { + loff_t pos = iocb->ki_pos; + task_io_account_write(res); if (!is_sync_kiocb(iocb)) { res = fuse_direct_IO(iocb, from); @@ -1822,7 +1823,7 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) /* * As in generic_file_direct_write(), invalidate after * write, to invalidate read-ahead cache that may have - * with the write. + * competed with the write. */ invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT, diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 85f738c531225a..c8d4c5f3af7e8c 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -1054,6 +1054,8 @@ u64 fuse_time_to_jiffies(u64 sec, u32 nsec); void fuse_change_entry_timeout(struct dentry *entry, struct fuse_entry_out *o); +void fuse_dentry_set_epoch(struct dentry *dentry, u64 epoch); + /* * Initialize fuse_conn */ diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index ce4891a04a3789..e89e2b0a57a6a5 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -791,6 +791,9 @@ static int fuse_opt_fd(struct fs_context *fsc, struct file *file) { struct fuse_fs_context *ctx = fsc->fs_private; + if (ctx->fud) + return invalfc(fsc, "Multiple fd specified"); + if (file->f_op != &fuse_dev_operations) return invalfc(fsc, "fd is not a fuse device"); /* @@ -1476,10 +1479,6 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) flags |= FUSE_PASSTHROUGH; - /* - * This is just an information flag for fuse server. No need to check - * the reply - server is either sending IORING_OP_URING_CMD or not. - */ if (fuse_uring_enabled()) flags |= FUSE_OVER_IO_URING; @@ -1641,6 +1640,8 @@ static int fuse_fill_super_submount(struct super_block *sb, fuse_fill_attr_from_inode(&root_attr, parent_fi); root = fuse_iget(sb, parent_fi->nodeid, 0, &root_attr, 0, 0, fuse_get_evict_ctr(fm->fc)); + if (!root) + return -ENOMEM; /* * This inode is just a duplicate, so it is not looked up and * its nlookup should not be incremented. fuse_iget() does diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index 0e13214917473e..5ca87151d70d98 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -260,7 +260,7 @@ retry: } if (fc->readdirplus_auto) set_bit(FUSE_I_INIT_RDPLUS, &get_fuse_inode(inode)->state); - dentry->d_time = epoch; + fuse_dentry_set_epoch(dentry, epoch); fuse_change_entry_timeout(dentry, o); dput(dentry); diff --git a/fs/hfsplus/catalog.c b/fs/hfsplus/catalog.c index 776ce36cf076b4..fe11c4b6dd997c 100644 --- a/fs/hfsplus/catalog.c +++ b/fs/hfsplus/catalog.c @@ -204,7 +204,7 @@ int hfsplus_find_cat(struct super_block *sb, u32 cnid, return err; type = be16_to_cpu(tmp.type); - if (type != HFSPLUS_FOLDER_THREAD && type != HFSPLUS_FILE_THREAD) { + if (!is_hfs_thread_record_type(type)) { pr_err("found bad thread record in catalog\n"); return -EIO; } @@ -350,23 +350,22 @@ int hfsplus_delete_cat(u32 cnid, struct inode *dir, const struct qstr *str) goto out; if (!str) { - int len; + hfsplus_cat_entry entry = {0}; hfsplus_cat_build_key_with_cnid(sb, fd.search_key, cnid); - err = hfs_brec_find(&fd, hfs_find_rec_by_key); + err = hfsplus_brec_read_cat(&fd, &entry); if (err) goto out; - off = fd.entryoffset + - offsetof(struct hfsplus_cat_thread, nodeName); - fd.search_key->cat.parent = cpu_to_be32(dir->i_ino); - hfs_bnode_read(fd.bnode, - &fd.search_key->cat.name.length, off, 2); - len = be16_to_cpu(fd.search_key->cat.name.length) * 2; - hfs_bnode_read(fd.bnode, - &fd.search_key->cat.name.unicode, - off + 2, len); - fd.search_key->key_len = cpu_to_be16(6 + len); + type = be16_to_cpu(entry.type); + if (!is_hfs_thread_record_type(type)) { + pr_err("found bad thread record in catalog\n"); + err = -EIO; + goto out; + } + + hfsplus_cat_build_key_uni(fd.search_key, dir->i_ino, + &entry.thread.nodeName); } else { err = hfsplus_cat_build_key(sb, fd.search_key, dir->i_ino, str); if (unlikely(err)) diff --git a/fs/hfsplus/hfsplus_fs.h b/fs/hfsplus/hfsplus_fs.h index ec04b82ad92787..7c8667d5a49c1d 100644 --- a/fs/hfsplus/hfsplus_fs.h +++ b/fs/hfsplus/hfsplus_fs.h @@ -521,6 +521,12 @@ static inline u32 hfsplus_cat_thread_size(const struct hfsplus_cat_thread *threa be16_to_cpu(thread->nodeName.length) * sizeof(hfsplus_unichr); } +static inline +bool is_hfs_thread_record_type(u16 type) +{ + return type == HFSPLUS_FOLDER_THREAD || type == HFSPLUS_FILE_THREAD; +} + int hfsplus_brec_read_cat(struct hfs_find_data *fd, hfsplus_cat_entry *entry); /* diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 216e1a0dd0b237..6a1d7e778cb0eb 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -1419,7 +1419,8 @@ hugetlbfs_fill_super(struct super_block *sb, struct fs_context *fc) goto out_free; return 0; out_free: - kfree(sbinfo->spool); + if (sbinfo->spool) + hugepage_put_subpool(sbinfo->spool); kfree(sbinfo); return -ENOMEM; } diff --git a/fs/inode.c b/fs/inode.c index 31c5b9ee3a81df..95e981b4e19c9e 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -2833,8 +2833,8 @@ struct timespec64 inode_set_ctime_to_ts(struct inode *inode, struct timespec64 t { trace_inode_set_ctime_to_ts(inode, &ts); set_normalized_timespec64(&ts, ts.tv_sec, ts.tv_nsec); - inode->i_ctime_sec = ts.tv_sec; - inode->i_ctime_nsec = ts.tv_nsec; + WRITE_ONCE(inode->i_ctime_sec, ts.tv_sec); + WRITE_ONCE(inode->i_ctime_nsec, ts.tv_nsec); return ts; } EXPORT_SYMBOL(inode_set_ctime_to_ts); @@ -2908,7 +2908,7 @@ struct timespec64 inode_set_ctime_current(struct inode *inode) */ cns = smp_load_acquire(&inode->i_ctime_nsec); if (cns & I_CTIME_QUERIED) { - struct timespec64 ctime = { .tv_sec = inode->i_ctime_sec, + struct timespec64 ctime = { .tv_sec = inode_get_ctime_sec(inode), .tv_nsec = cns & ~I_CTIME_QUERIED }; if (timespec64_compare(&now, &ctime) <= 0) { @@ -2920,7 +2920,7 @@ struct timespec64 inode_set_ctime_current(struct inode *inode) mgtime_counter_inc(mg_ctime_updates); /* No need to cmpxchg if it's exactly the same */ - if (cns == now.tv_nsec && inode->i_ctime_sec == now.tv_sec) { + if (cns == now.tv_nsec && inode_get_ctime_sec(inode) == now.tv_sec) { trace_ctime_xchg_skip(inode, &now); goto out; } @@ -2929,7 +2929,7 @@ retry: /* Try to swap the nsec value into place. */ if (try_cmpxchg(&inode->i_ctime_nsec, &cur, now.tv_nsec)) { /* If swap occurred, then we're (mostly) done */ - inode->i_ctime_sec = now.tv_sec; + WRITE_ONCE(inode->i_ctime_sec, now.tv_sec); trace_ctime_ns_xchg(inode, cns, now.tv_nsec, cur); mgtime_counter_inc(mg_ctime_swaps); } else { @@ -2944,7 +2944,7 @@ retry: goto retry; } /* Otherwise, keep the existing ctime */ - now.tv_sec = inode->i_ctime_sec; + now.tv_sec = inode_get_ctime_sec(inode); now.tv_nsec = cur & ~I_CTIME_QUERIED; } out: @@ -2977,7 +2977,7 @@ struct timespec64 inode_set_ctime_deleg(struct inode *inode, struct timespec64 u /* pairs with try_cmpxchg below */ cur = smp_load_acquire(&inode->i_ctime_nsec); cur_ts.tv_nsec = cur & ~I_CTIME_QUERIED; - cur_ts.tv_sec = inode->i_ctime_sec; + cur_ts.tv_sec = inode_get_ctime_sec(inode); /* If the update is older than the existing value, skip it. */ if (timespec64_compare(&update, &cur_ts) <= 0) @@ -3003,7 +3003,7 @@ struct timespec64 inode_set_ctime_deleg(struct inode *inode, struct timespec64 u retry: old = cur; if (try_cmpxchg(&inode->i_ctime_nsec, &cur, update.tv_nsec)) { - inode->i_ctime_sec = update.tv_sec; + WRITE_ONCE(inode->i_ctime_sec, update.tv_sec); mgtime_counter_inc(mg_ctime_swaps); return update; } @@ -3019,7 +3019,7 @@ retry: goto retry; /* Otherwise, it was a new timestamp. */ - cur_ts.tv_sec = inode->i_ctime_sec; + cur_ts.tv_sec = inode_get_ctime_sec(inode); cur_ts.tv_nsec = cur & ~I_CTIME_QUERIED; return cur_ts; } diff --git a/fs/iomap/bio.c b/fs/iomap/bio.c index 30ef78a66b4f09..48100c614431c0 100644 --- a/fs/iomap/bio.c +++ b/fs/iomap/bio.c @@ -184,5 +184,6 @@ int iomap_bio_read_folio_range_sync(const struct iomap_iter *iter, error = fs_bio_integrity_verify(&bio, sector, len); fs_bio_integrity_free(&bio); } + bio_uninit(&bio); return error; } diff --git a/fs/iomap/iter.c b/fs/iomap/iter.c index e4a29829591a75..c445a38b6285dd 100644 --- a/fs/iomap/iter.c +++ b/fs/iomap/iter.c @@ -6,12 +6,19 @@ #include #include "trace.h" -static inline void iomap_iter_clean_fbatch(struct iomap_iter *iter) +/* + * Release the iter folio batch. Note that the iomap flag is meant to control + * the I/O path for the mapping and may not be set in error situations. + */ +static inline void iomap_iter_clean_fbatch(const struct iomap_iter *iter, + struct iomap *iomap) { - if (iter->iomap.flags & IOMAP_F_FOLIO_BATCH) { + if (!iter->fbatch) + return; + iomap->flags &= ~IOMAP_F_FOLIO_BATCH; + if (folio_batch_count(iter->fbatch)) { folio_batch_release(iter->fbatch); folio_batch_reinit(iter->fbatch); - iter->iomap.flags &= ~IOMAP_F_FOLIO_BATCH; } } @@ -40,51 +47,27 @@ static inline void iomap_iter_done(struct iomap_iter *iter) } /** - * iomap_iter - iterate over a ranges in a file - * @iter: iteration structue - * @ops: iomap ops provided by the file system + * iomap_iter_continue - decide whether iteration should continue + * @iter: iteration structure + * @iomap: the mapping that was just processed + * @srcmap: the source mapping that was just processed * - * Iterate over filesystem-provided space mappings for the provided file range. + * Helper normally called via iomap_iter_next(). Called after the previous + * mapping has been finished to determine whether there is more of the file + * range left to process. * - * This function handles cleanup of resources acquired for iteration when the - * filesystem indicates there are no more space mappings, which means that this - * function must be called in a loop that continues as long it returns a - * positive value. If 0 or a negative value is returned, the caller must not - * return to the loop body. Within a loop body, there are two ways to break out - * of the loop body: leave @iter.status unchanged, or set it to a negative - * errno. + * Returns 1 if there is more work to do, in which case @iomap and @srcmap are + * cleared so the caller can produce the next mapping; zero if the range is + * fully consumed; or a negative errno on error. */ -int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops) +int iomap_iter_continue(const struct iomap_iter *iter, struct iomap *iomap, + struct iomap *srcmap, int ret) { - bool stale = iter->iomap.flags & IOMAP_F_STALE; - ssize_t advanced; - u64 olen; - int ret; + const bool stale = iomap->flags & IOMAP_F_STALE; + const ssize_t advanced = iter->pos - iter->iter_start_pos; - trace_iomap_iter(iter, ops, _RET_IP_); - - if (!iter->iomap.length) - goto begin; - - /* - * Calculate how far the iter was advanced and the original length bytes - * for ->iomap_end(). - */ - advanced = iter->pos - iter->iter_start_pos; - olen = iter->len + advanced; - - if (ops->iomap_end) { - ret = ops->iomap_end(iter->inode, iter->iter_start_pos, - iomap_length_trim(iter, iter->iter_start_pos, - olen), - advanced, iter->flags, &iter->iomap); - if (ret < 0 && !advanced) - return ret; - } - - /* detect old return semantics where this would advance */ - if (WARN_ON_ONCE(iter->status > 0)) - iter->status = -EIO; + if (ret < 0 && !advanced) + return ret; /* * Use iter->len to determine whether to continue onto the next mapping. @@ -92,25 +75,60 @@ int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops) * advanced at all (i.e. no work was done for some reason) unless the * mapping has been marked stale and needs to be reprocessed. */ - if (iter->status < 0) + if (WARN_ON_ONCE(iter->status > 0)) + /* detect old return semantics where this would advance */ + ret = -EIO; + else if (iter->status < 0) ret = iter->status; else if (iter->len == 0 || (!advanced && !stale)) ret = 0; else ret = 1; - iomap_iter_clean_fbatch(iter); - iter->status = 0; + + iomap_iter_clean_fbatch(iter, iomap); + if (ret <= 0) return ret; - memset(&iter->iomap, 0, sizeof(iter->iomap)); - memset(&iter->srcmap, 0, sizeof(iter->srcmap)); + memset(iomap, 0, sizeof(*iomap)); + memset(srcmap, 0, sizeof(*srcmap)); -begin: - ret = ops->iomap_begin(iter->inode, iter->pos, iter->len, iter->flags, - &iter->iomap, &iter->srcmap); - if (ret < 0) - return ret; - iomap_iter_done(iter); - return 1; + return ret; +} +EXPORT_SYMBOL_GPL(iomap_iter_continue); + +/** + * iomap_iter - iterate over ranges in a file + * @iter: iteration structure + * @ops: iomap ops provided by the filesystem + * + * Iterate over filesystem-provided space mappings for the provided file range. + * + * This function handles cleanup of resources acquired for iteration when the + * filesystem indicates there are no more space mappings, which means that this + * function must be called in a loop that continues as long it returns a + * positive value. If 0 or a negative value is returned, the caller must not + * return to the loop body. Within a loop body, there are two ways to break out + * of the loop body: leave @iter.status unchanged, or set it to a negative + * errno. + */ +int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops) +{ + int ret; + + trace_iomap_iter(iter, ops, _RET_IP_); + + if (ops->iomap_next) + ret = ops->iomap_next(iter, &iter->iomap, &iter->srcmap); + else + ret = iomap_iter_next(iter, &iter->iomap, &iter->srcmap, + ops->iomap_begin, ops->iomap_end); + + iter->status = 0; + if (ret > 0) + iomap_iter_done(iter); + else if (ret < 0) + iomap_iter_clean_fbatch(iter, &iter->iomap); + + return ret; } diff --git a/fs/isofs/compress.c b/fs/isofs/compress.c index 397568b9c7e7d3..f9869d62b85096 100644 --- a/fs/isofs/compress.c +++ b/fs/isofs/compress.c @@ -65,12 +65,14 @@ static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start, /* Empty block? */ if (block_size == 0) { for ( i = 0 ; i < pcount ; i++ ) { + unsigned int off = i ? 0 : poffset; + if (!pages[i]) continue; - memzero_page(pages[i], 0, PAGE_SIZE); + memzero_page(pages[i], off, PAGE_SIZE - off); SetPageUptodate(pages[i]); } - return ((loff_t)pcount) << PAGE_SHIFT; + return (((loff_t)pcount) << PAGE_SHIFT) - poffset; } /* Because zlib is not thread-safe, do all the I/O at the top. */ @@ -291,6 +293,7 @@ static int zisofs_fill_pages(struct inode *inode, int full_page, int pcount, memzero_page(*pages, poffset, PAGE_SIZE - poffset); SetPageUptodate(*pages); } + brelse(bh); return 0; } diff --git a/fs/lockd/nlm3xdr_gen.c b/fs/lockd/nlm3xdr_gen.c index 9ed5a41b5daf37..df14692ce37f2b 100644 --- a/fs/lockd/nlm3xdr_gen.c +++ b/fs/lockd/nlm3xdr_gen.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 // Generated by xdrgen. Manual edits will be lost. // XDR specification file: ../../Documentation/sunrpc/xdr/nlm3.x -// XDR specification modification time: Thu Apr 23 10:56:34 2026 +// XDR specification modification time: Mon Jun 29 20:42:29 2026 #include diff --git a/fs/lockd/nlm3xdr_gen.h b/fs/lockd/nlm3xdr_gen.h index c99038e99805f1..3824ffe2aae460 100644 --- a/fs/lockd/nlm3xdr_gen.h +++ b/fs/lockd/nlm3xdr_gen.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Generated by xdrgen. Manual edits will be lost. */ /* XDR specification file: ../../Documentation/sunrpc/xdr/nlm3.x */ -/* XDR specification modification time: Thu Apr 23 10:56:34 2026 */ +/* XDR specification modification time: Mon Jun 29 20:42:29 2026 */ #ifndef _LINUX_XDRGEN_NLM3_DECL_H #define _LINUX_XDRGEN_NLM3_DECL_H diff --git a/fs/lockd/nlm4xdr_gen.c b/fs/lockd/nlm4xdr_gen.c index 1c8c221db456cc..5a60aff0671475 100644 --- a/fs/lockd/nlm4xdr_gen.c +++ b/fs/lockd/nlm4xdr_gen.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 // Generated by xdrgen. Manual edits will be lost. // XDR specification file: ../../Documentation/sunrpc/xdr/nlm4.x -// XDR specification modification time: Thu Dec 25 13:10:19 2025 +// XDR specification modification time: Mon Jun 29 20:42:29 2026 #include @@ -20,6 +20,16 @@ xdrgen_decode_fsh4_mode(struct xdr_stream *xdr, fsh4_mode *ptr) if (xdr_stream_decode_u32(xdr, &val) < 0) return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case fsm_DN: + case fsm_DR: + case fsm_DW: + case fsm_DRW: + break; + default: + return false; + } *ptr = val; return true; } @@ -31,6 +41,16 @@ xdrgen_decode_fsh4_access(struct xdr_stream *xdr, fsh4_access *ptr) if (xdr_stream_decode_u32(xdr, &val) < 0) return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case fsa_NONE: + case fsa_R: + case fsa_W: + case fsa_RW: + break; + default: + return false; + } *ptr = val; return true; } @@ -62,7 +82,30 @@ xdrgen_decode_int32(struct xdr_stream *xdr, int32 *ptr) static bool __maybe_unused xdrgen_decode_nlm4_stats(struct xdr_stream *xdr, nlm4_stats *ptr) { - return xdr_stream_decode_be32(xdr, ptr) == 0; + __be32 raw; + u32 val; + + if (xdr_stream_decode_be32(xdr, &raw) < 0) + return false; + val = be32_to_cpu(raw); + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case NLM4_GRANTED: + case NLM4_DENIED: + case NLM4_DENIED_NOLOCKS: + case NLM4_BLOCKED: + case NLM4_DENIED_GRACE_PERIOD: + case NLM4_DEADLCK: + case NLM4_ROFS: + case NLM4_STALE_FH: + case NLM4_FBIG: + case NLM4_FAILED: + break; + default: + return false; + } + *ptr = raw; + return true; } static bool __maybe_unused diff --git a/fs/lockd/nlm4xdr_gen.h b/fs/lockd/nlm4xdr_gen.h index b6008b296a3e3b..ce9dda0a052a30 100644 --- a/fs/lockd/nlm4xdr_gen.h +++ b/fs/lockd/nlm4xdr_gen.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Generated by xdrgen. Manual edits will be lost. */ /* XDR specification file: ../../Documentation/sunrpc/xdr/nlm4.x */ -/* XDR specification modification time: Thu Dec 25 13:10:19 2025 */ +/* XDR specification modification time: Mon Jun 29 20:42:29 2026 */ #ifndef _LINUX_XDRGEN_NLM4_DECL_H #define _LINUX_XDRGEN_NLM4_DECL_H diff --git a/fs/nfs/blocklayout/dev.c b/fs/nfs/blocklayout/dev.c index bb35f88501cea5..db4bb0a6283e51 100644 --- a/fs/nfs/blocklayout/dev.c +++ b/fs/nfs/blocklayout/dev.c @@ -85,15 +85,17 @@ bl_free_device(struct pnfs_block_dev *dev) { bl_unregister_dev(dev); - if (dev->nr_children) { + if (dev->children) { int i; for (i = 0; i < dev->nr_children; i++) bl_free_device(&dev->children[i]); kfree(dev->children); - } else { - if (dev->bdev_file) - fput(dev->bdev_file); + dev->children = NULL; + dev->nr_children = 0; + } else if (dev->bdev_file) { + fput(dev->bdev_file); + dev->bdev_file = NULL; } } @@ -437,6 +439,7 @@ bl_parse_scsi(struct nfs_server *server, struct pnfs_block_dev *d, out_blkdev_put: fput(d->bdev_file); + d->bdev_file = NULL; return error; } @@ -472,8 +475,11 @@ bl_parse_concat(struct nfs_server *server, struct pnfs_block_dev *d, for (i = 0; i < v->concat.volumes_count; i++) { ret = bl_parse_deviceid(server, &d->children[i], volumes, v->concat.volumes[i], gfp_mask); - if (ret) + if (ret) { + bl_free_device(&d->children[i]); + bl_free_device(d); return ret; + } d->nr_children++; d->children[i].start += len; @@ -501,8 +507,11 @@ bl_parse_stripe(struct nfs_server *server, struct pnfs_block_dev *d, for (i = 0; i < v->stripe.volumes_count; i++) { ret = bl_parse_deviceid(server, &d->children[i], volumes, v->stripe.volumes[i], gfp_mask); - if (ret) + if (ret) { + bl_free_device(&d->children[i]); + bl_free_device(d); return ret; + } d->nr_children++; len += d->children[i].len; diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c index 4382baddc9eed2..a6406707ede6c0 100644 --- a/fs/nfs/callback_xdr.c +++ b/fs/nfs/callback_xdr.c @@ -417,7 +417,7 @@ static __be32 decode_cb_sequence_args(struct svc_rqst *rqstp, args->csa_nrclists = ntohl(*p++); args->csa_rclists = NULL; if (args->csa_nrclists) { - args->csa_rclists = kmalloc_objs(*args->csa_rclists, + args->csa_rclists = kzalloc_objs(*args->csa_rclists, args->csa_nrclists); if (unlikely(args->csa_rclists == NULL)) return htonl(NFS4ERR_RESOURCE); diff --git a/fs/nfs/client.c b/fs/nfs/client.c index 4dcb91ab30397f..60386330aeeca2 100644 --- a/fs/nfs/client.c +++ b/fs/nfs/client.c @@ -215,9 +215,21 @@ static void nfs_cb_idr_remove_locked(struct nfs_client *clp) { struct nfs_net *nn = net_generic(clp->cl_net, nfs_net_id); - if (clp->cl_cb_ident) + if (clp->cl_cb_ident) { idr_remove(&nn->cb_ident_idr, clp->cl_cb_ident); + clp->cl_cb_ident = 0; + } +} + +void nfs_cb_idr_remove(struct nfs_client *clp) +{ + struct nfs_net *nn = net_generic(clp->cl_net, nfs_net_id); + + spin_lock(&nn->nfs_client_lock); + nfs_cb_idr_remove_locked(clp); + spin_unlock(&nn->nfs_client_lock); } +EXPORT_SYMBOL_GPL(nfs_cb_idr_remove); static void pnfs_init_server(struct nfs_server *server) { diff --git a/fs/nfs/delegation.c b/fs/nfs/delegation.c index 9546d2195c25f5..ab3e441464a66e 100644 --- a/fs/nfs/delegation.c +++ b/fs/nfs/delegation.c @@ -447,11 +447,14 @@ int nfs_inode_set_delegation(struct inode *inode, const struct cred *cred, struct nfs_inode *nfsi = NFS_I(inode); struct nfs_delegation *delegation, *old_delegation; struct nfs_delegation *freeme = NULL; + bool orphaned = false; int status = 0; delegation = kmalloc_obj(*delegation, GFP_KERNEL_ACCOUNT); - if (delegation == NULL) + if (delegation == NULL) { + nfs4_proc_delegreturn(inode, cred, stateid, NULL, 0); return -ENOMEM; + } nfs4_stateid_copy(&delegation->stateid, stateid); refcount_set(&delegation->refcount, 1); delegation->type = type; @@ -500,11 +503,15 @@ int nfs_inode_set_delegation(struct inode *inode, const struct cred *cred, goto out; } if (test_and_set_bit(NFS_DELEGATION_RETURNING, - &old_delegation->flags)) + &old_delegation->flags)) { + orphaned = true; goto out; + } } - if (!nfs_detach_delegations_locked(nfsi, old_delegation, clp)) + if (!nfs_detach_delegations_locked(nfsi, old_delegation, clp)) { + orphaned = true; goto out; + } freeme = old_delegation; add_new: /* @@ -539,8 +546,11 @@ add_new: nfs_update_delegated_mtime(inode); out: spin_unlock(&clp->cl_lock); - if (delegation != NULL) + if (delegation != NULL) { + if (orphaned) + nfs_do_return_delegation(inode, delegation, 0); __nfs_free_delegation(delegation); + } if (freeme != NULL) { nfs_do_return_delegation(inode, freeme, 0); nfs_mark_delegation_revoked(server, freeme); @@ -594,7 +604,7 @@ delay: spin_lock(&server->delegations_lock); if (list_empty(&delegation->entry)) refcount_inc(&delegation->refcount); - list_move_tail(&delegation->entry, &server->delegations_return); + list_move_tail(&delegation->entry, &server->delegations_delayed); spin_unlock(&server->delegations_lock); set_bit(NFS4CLNT_DELEGRETURN_DELAYED, &server->nfs_client->cl_state); abort: diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index c7caffb31935c7..e97ce5240c18ed 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -2319,6 +2319,13 @@ int nfs_atomic_open_v23(struct inode *dir, struct dentry *dentry, if (open_flags & O_CREAT) { error = nfs_do_create(dir, dentry, mode, open_flags); if (!error) { + /* With UNCHECKED mode, a server may return NFS3_OK for + * a pre-existing non-regular file (e.g. a symlink). + * Let the VFS handle it; calling finish_open() would + * hit no_open() and return -ENXIO. + */ + if (!d_is_reg(dentry)) + return finish_no_open(file, NULL); file->f_mode |= FMODE_CREATED; return finish_open(file, dentry, NULL); } else if (error != -EEXIST || open_flags & O_EXCL) diff --git a/fs/nfs/filelayout/filelayoutdev.c b/fs/nfs/filelayout/filelayoutdev.c index 7226989ee4d536..9ddcaf86fcef08 100644 --- a/fs/nfs/filelayout/filelayoutdev.c +++ b/fs/nfs/filelayout/filelayoutdev.c @@ -170,7 +170,8 @@ nfs4_fl_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev, goto out_err_free_deviceid; } - dsaddr->ds_list[i] = nfs4_pnfs_ds_add(net, &dsaddrs, gfp_flags); + dsaddr->ds_list[i] = nfs4_pnfs_ds_add(net, &dsaddrs, 4, + gfp_flags); if (!dsaddr->ds_list[i]) goto out_err_drain_dsaddrs; trace_fl_getdevinfo(server, &pdev->dev_id, dsaddr->ds_list[i]->ds_remotestr); diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index c4aa995026f671..ef26fcab9c1039 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -1322,7 +1322,8 @@ static int ff_layout_async_handle_error_v4(struct rpc_task *task, struct pnfs_layout_hdr *lo = lseg->pls_layout; struct inode *inode = lo->plh_inode; struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx, dss_id); - struct nfs4_slot_table *tbl = &clp->cl_session->fc_slot_table; + struct nfs4_slot_table *tbl = nfs4_has_session(clp) ? + &clp->cl_session->fc_slot_table : clp->cl_slot_tbl; switch (op_status) { case NFS4_OK: diff --git a/fs/nfs/flexfilelayout/flexfilelayoutdev.c b/fs/nfs/flexfilelayout/flexfilelayoutdev.c index 1109462a969913..c716fd99e8ef9f 100644 --- a/fs/nfs/flexfilelayout/flexfilelayoutdev.c +++ b/fs/nfs/flexfilelayout/flexfilelayoutdev.c @@ -153,7 +153,8 @@ nfs4_ff_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev, new_ds->ds_versions = ds_versions; new_ds->ds_versions_cnt = version_count; - new_ds->ds = nfs4_pnfs_ds_add(net, &dsaddrs, gfp_flags); + new_ds->ds = nfs4_pnfs_ds_add(net, &dsaddrs, ds_versions[0].version, + gfp_flags); if (!new_ds->ds) goto out_err_drain_dsaddrs; diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index 5bcd4027d2033e..e538736bb165a0 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -1531,9 +1531,7 @@ int nfs_clear_invalid_mapping(struct address_space *mapping) ret = nfs_invalidate_mapping(inode, mapping); trace_nfs_invalidate_mapping_exit(inode, ret); - clear_bit_unlock(NFS_INO_INVALIDATING, bitlock); - smp_mb__after_atomic(); - wake_up_bit(bitlock, NFS_INO_INVALIDATING); + clear_and_wake_up_bit(NFS_INO_INVALIDATING, bitlock); out: return ret; } diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index e4533f5836321d..864fa092bcead1 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -225,6 +225,7 @@ void nfs_server_copy_userdata(struct nfs_server *, struct nfs_server *); extern void nfs_put_client(struct nfs_client *); extern void nfs_free_client(struct nfs_client *); +void nfs_cb_idr_remove(struct nfs_client *clp); extern struct nfs_client *nfs4_find_client_ident(struct net *, int); extern struct nfs_client * nfs4_find_client_sessionid(struct net *, const struct sockaddr *, diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c index 819e435c3a42b7..5bc6dc9c1135ac 100644 --- a/fs/nfs/nfs4client.c +++ b/fs/nfs/nfs4client.c @@ -261,6 +261,7 @@ struct nfs_client *nfs4_alloc_client(const struct nfs_client_initdata *cl_init) return clp; error: + nfs_cb_idr_remove(clp); nfs_free_client(clp); return ERR_PTR(err); } diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 5709c6fea85bbd..2edd16e4981a21 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -7762,6 +7762,7 @@ static int nfs4_add_lease(struct file *file, int arg, struct file_lease **lease, { struct inode *inode = file_inode(file); fmode_t type = arg == F_RDLCK ? FMODE_READ : FMODE_WRITE; + fl_owner_t owner = (*lease)->c.flc_owner; int ret; /* No delegation, no lease */ @@ -7771,7 +7772,8 @@ static int nfs4_add_lease(struct file *file, int arg, struct file_lease **lease, if (ret || nfs4_have_delegation(inode, type, 0)) return ret; /* We raced with a delegation return */ - nfs4_delete_lease(file, priv); + dprintk("%s: raced with a delegation return\n", __func__); + nfs4_delete_lease(file, &owner); return -EAGAIN; } diff --git a/fs/nfs/nfs4session.c b/fs/nfs/nfs4session.c index 5c128957a0a41a..993f0db7cf5e4d 100644 --- a/fs/nfs/nfs4session.c +++ b/fs/nfs/nfs4session.c @@ -632,16 +632,22 @@ int nfs4_init_ds_session(struct nfs_client *clp, unsigned long lease_time) int ret; spin_lock(&clp->cl_lock); - if (test_and_clear_bit(NFS4_SESSION_INITING, &session->session_state)) { - /* - * Do not set NFS_CS_CHECK_LEASE_TIME instead set the - * DS lease to be equal to the MDS lease. - */ + /* + * Do not set NFS_CS_CHECK_LEASE_TIME instead set the + * DS lease to be equal to the MDS lease. + * + * A v4.0 DS has no session, so seed the lease every time. + */ + if (!session || + test_and_clear_bit(NFS4_SESSION_INITING, &session->session_state)) { clp->cl_lease_time = lease_time; clp->cl_last_renewal = jiffies; } spin_unlock(&clp->cl_lock); + if (!session) + return 0; + ret = nfs41_check_session_ready(clp); if (ret) return ret; diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index b1a30a16287f3e..fe5af86f9c486e 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -2100,15 +2100,6 @@ static bool pnfs_is_first_layoutget(struct pnfs_layout_hdr *lo) return test_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); } -static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo) -{ - unsigned long *bitlock = &lo->plh_flags; - - clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock); - smp_mb__after_atomic(); - wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET); -} - static void _add_to_server_list(struct pnfs_layout_hdr *lo, struct nfs_server *server) { @@ -2284,7 +2275,8 @@ lookup_again: iomode, lo, lseg, PNFS_UPDATE_LAYOUT_INVALID_OPEN); nfs4_schedule_stateid_recovery(server, ctx->state); - pnfs_clear_first_layoutget(lo); + clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, + &lo->plh_flags); pnfs_put_layout_hdr(lo); goto lookup_again; } @@ -2353,7 +2345,8 @@ lookup_again: if (!exception.retry) goto out_put_layout_hdr; if (first) - pnfs_clear_first_layoutget(lo); + clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, + &lo->plh_flags); trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY); pnfs_put_layout_hdr(lo); @@ -2365,7 +2358,7 @@ lookup_again: out_put_layout_hdr: if (first) - pnfs_clear_first_layoutget(lo); + clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, PNFS_UPDATE_LAYOUT_EXIT); pnfs_put_layout_hdr(lo); @@ -2457,7 +2450,7 @@ static void _lgopen_prepare_attached(struct nfs4_opendata *data, lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, nfs_io_gfp_mask()); if (!lgp) { - pnfs_clear_first_layoutget(lo); + clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); nfs_layoutget_end(lo); pnfs_put_layout_hdr(lo); return; @@ -2561,7 +2554,8 @@ void nfs4_lgopen_release(struct nfs4_layoutget *lgp) { if (lgp != NULL) { if (lgp->lo) { - pnfs_clear_first_layoutget(lgp->lo); + clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, + &lgp->lo->plh_flags); nfs_layoutget_end(lgp->lo); } pnfs_layoutget_free(lgp); @@ -3274,15 +3268,6 @@ pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc) } EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages); -static void pnfs_clear_layoutcommitting(struct inode *inode) -{ - unsigned long *bitlock = &NFS_I(inode)->flags; - - clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock); - smp_mb__after_atomic(); - wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING); -} - /* * There can be multiple RW segments. */ @@ -3307,7 +3292,7 @@ static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *lis pnfs_put_lseg(lseg); } - pnfs_clear_layoutcommitting(inode); + clear_and_wake_up_bit(NFS_INO_LAYOUTCOMMITTING, &NFS_I(inode)->flags); } void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg) @@ -3447,7 +3432,7 @@ out_unlock: spin_unlock(&inode->i_lock); kfree(data); clear_layoutcommitting: - pnfs_clear_layoutcommitting(inode); + clear_and_wake_up_bit(NFS_INO_LAYOUTCOMMITTING, &NFS_I(inode)->flags); goto out; } EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode); diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index eb39859c216c97..f06a03ac4fa375 100644 --- a/fs/nfs/pnfs.h +++ b/fs/nfs/pnfs.h @@ -63,6 +63,7 @@ struct nfs4_pnfs_ds { const struct net *ds_net; struct nfs_client *ds_clp; refcount_t ds_count; + u32 ds_version; /* cache key, with ds_addrs */ unsigned long ds_state; #define NFS4DS_CONNECTING 0 /* ds is establishing connection */ }; @@ -417,7 +418,7 @@ void pnfs_generic_write_commit_done(struct rpc_task *task, void *data); void nfs4_pnfs_ds_put(struct nfs4_pnfs_ds *ds); struct nfs4_pnfs_ds *nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, - gfp_t gfp_flags); + u32 version, gfp_t gfp_flags); void nfs4_pnfs_v3_ds_connect_unload(void); int nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds, struct nfs4_deviceid_node *devid, unsigned int timeo, diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index 648c95b78eeab7..49c621cde43a86 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -603,15 +603,17 @@ _same_data_server_addrs_locked(const struct list_head *dsaddrs1, } /* - * Lookup DS by addresses. nfs4_ds_cache_lock is held + * Lookup DS by addresses and NFS version. nfs4_ds_cache_lock is held */ static struct nfs4_pnfs_ds * -_data_server_lookup_locked(const struct nfs_net *nn, const struct list_head *dsaddrs) +_data_server_lookup_locked(const struct nfs_net *nn, + const struct list_head *dsaddrs, u32 version) { struct nfs4_pnfs_ds *ds; list_for_each_entry(ds, &nn->nfs4_data_server_cache, ds_node) - if (_same_data_server_addrs_locked(&ds->ds_addrs, dsaddrs)) + if (ds->ds_version == version && + _same_data_server_addrs_locked(&ds->ds_addrs, dsaddrs)) return ds; return NULL; } @@ -719,7 +721,8 @@ out_err: * uncached and return cached struct nfs4_pnfs_ds. */ struct nfs4_pnfs_ds * -nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, gfp_t gfp_flags) +nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, u32 version, + gfp_t gfp_flags) { struct nfs_net *nn = net_generic(net, nfs_net_id); struct nfs4_pnfs_ds *tmp_ds, *ds = NULL; @@ -738,7 +741,7 @@ nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, gfp_t gfp_fla remotestr = nfs4_pnfs_remotestr(dsaddrs, gfp_flags); spin_lock(&nn->nfs4_data_server_lock); - tmp_ds = _data_server_lookup_locked(nn, dsaddrs); + tmp_ds = _data_server_lookup_locked(nn, dsaddrs, version); if (tmp_ds == NULL) { INIT_LIST_HEAD(&ds->ds_addrs); list_splice_init(dsaddrs, &ds->ds_addrs); @@ -747,6 +750,7 @@ nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, gfp_t gfp_fla INIT_LIST_HEAD(&ds->ds_node); ds->ds_net = net; ds->ds_clp = NULL; + ds->ds_version = version; list_add(&ds->ds_node, &nn->nfs4_data_server_cache); dprintk("%s add new data server %s\n", __func__, ds->ds_remotestr); diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 6180f6dc8995b4..258621750eb7f5 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -1276,9 +1276,9 @@ out_dec: void nfs4_put_stid(struct nfs4_stid *s) { - struct svc_export *exp = s->sc_export; struct nfs4_file *fp = s->sc_file; struct nfs4_client *clp = s->sc_client; + struct svc_export *exp; might_lock(&clp->cl_lock); @@ -1289,6 +1289,8 @@ nfs4_put_stid(struct nfs4_stid *s) idr_remove(&clp->cl_stateids, s->sc_stateid.si_opaque.so_id); if (s->sc_status & SC_STATUS_ADMIN_REVOKED) atomic_dec(&s->sc_client->cl_admin_revoked); + /* Read under cl_lock to serialize with drop_stid_export(). */ + exp = s->sc_export; nfs4_free_cpntf_statelist(clp->net, s); spin_unlock(&clp->cl_lock); s->sc_free(s); @@ -1745,6 +1747,7 @@ static void free_ol_stateid_reaplist(struct list_head *reaplist) { struct nfs4_ol_stateid *stp; + struct svc_export *exp; struct nfs4_file *fp; might_sleep(); @@ -1754,7 +1757,10 @@ free_ol_stateid_reaplist(struct list_head *reaplist) st_locks); list_del(&stp->st_locks); fp = stp->st_stid.sc_file; + exp = stp->st_stid.sc_export; stp->st_stid.sc_free(&stp->st_stid); + if (exp) + exp_put(exp); if (fp) put_nfs4_file(fp); } diff --git a/fs/nfsd/nfs4xdr_gen.c b/fs/nfsd/nfs4xdr_gen.c index 824497051b876a..d92dbae5db0fc7 100644 --- a/fs/nfsd/nfs4xdr_gen.c +++ b/fs/nfsd/nfs4xdr_gen.c @@ -1,16 +1,16 @@ // SPDX-License-Identifier: GPL-2.0 // Generated by xdrgen. Manual edits will be lost. // XDR specification file: ../../Documentation/sunrpc/xdr/nfs4_1.x -// XDR specification modification time: Thu Jan 8 23:12:07 2026 +// XDR specification modification time: Tue Jun 30 11:57:21 2026 #include #include "nfs4xdr_gen.h" static bool __maybe_unused -xdrgen_decode_int64_t(struct xdr_stream *xdr, int64_t *ptr) +xdrgen_decode_int32_t(struct xdr_stream *xdr, int32_t *ptr) { - return xdrgen_decode_hyper(xdr, ptr); + return xdrgen_decode_int(xdr, ptr); } static bool __maybe_unused @@ -19,6 +19,154 @@ xdrgen_decode_uint32_t(struct xdr_stream *xdr, uint32_t *ptr) return xdrgen_decode_unsigned_int(xdr, ptr); } +static bool __maybe_unused +xdrgen_decode_int64_t(struct xdr_stream *xdr, int64_t *ptr) +{ + return xdrgen_decode_hyper(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_uint64_t(struct xdr_stream *xdr, uint64_t *ptr) +{ + return xdrgen_decode_unsigned_hyper(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_nfsstat4(struct xdr_stream *xdr, nfsstat4 *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case NFS4_OK: + case NFS4ERR_PERM: + case NFS4ERR_NOENT: + case NFS4ERR_IO: + case NFS4ERR_NXIO: + case NFS4ERR_ACCESS: + case NFS4ERR_EXIST: + case NFS4ERR_XDEV: + case NFS4ERR_NOTDIR: + case NFS4ERR_ISDIR: + case NFS4ERR_INVAL: + case NFS4ERR_FBIG: + case NFS4ERR_NOSPC: + case NFS4ERR_ROFS: + case NFS4ERR_MLINK: + case NFS4ERR_NAMETOOLONG: + case NFS4ERR_NOTEMPTY: + case NFS4ERR_DQUOT: + case NFS4ERR_STALE: + case NFS4ERR_BADHANDLE: + case NFS4ERR_BAD_COOKIE: + case NFS4ERR_NOTSUPP: + case NFS4ERR_TOOSMALL: + case NFS4ERR_SERVERFAULT: + case NFS4ERR_BADTYPE: + case NFS4ERR_DELAY: + case NFS4ERR_SAME: + case NFS4ERR_DENIED: + case NFS4ERR_EXPIRED: + case NFS4ERR_LOCKED: + case NFS4ERR_GRACE: + case NFS4ERR_FHEXPIRED: + case NFS4ERR_SHARE_DENIED: + case NFS4ERR_WRONGSEC: + case NFS4ERR_CLID_INUSE: + case NFS4ERR_RESOURCE: + case NFS4ERR_MOVED: + case NFS4ERR_NOFILEHANDLE: + case NFS4ERR_MINOR_VERS_MISMATCH: + case NFS4ERR_STALE_CLIENTID: + case NFS4ERR_STALE_STATEID: + case NFS4ERR_OLD_STATEID: + case NFS4ERR_BAD_STATEID: + case NFS4ERR_BAD_SEQID: + case NFS4ERR_NOT_SAME: + case NFS4ERR_LOCK_RANGE: + case NFS4ERR_SYMLINK: + case NFS4ERR_RESTOREFH: + case NFS4ERR_LEASE_MOVED: + case NFS4ERR_ATTRNOTSUPP: + case NFS4ERR_NO_GRACE: + case NFS4ERR_RECLAIM_BAD: + case NFS4ERR_RECLAIM_CONFLICT: + case NFS4ERR_BADXDR: + case NFS4ERR_LOCKS_HELD: + case NFS4ERR_OPENMODE: + case NFS4ERR_BADOWNER: + case NFS4ERR_BADCHAR: + case NFS4ERR_BADNAME: + case NFS4ERR_BAD_RANGE: + case NFS4ERR_LOCK_NOTSUPP: + case NFS4ERR_OP_ILLEGAL: + case NFS4ERR_DEADLOCK: + case NFS4ERR_FILE_OPEN: + case NFS4ERR_ADMIN_REVOKED: + case NFS4ERR_CB_PATH_DOWN: + case NFS4ERR_BADIOMODE: + case NFS4ERR_BADLAYOUT: + case NFS4ERR_BAD_SESSION_DIGEST: + case NFS4ERR_BADSESSION: + case NFS4ERR_BADSLOT: + case NFS4ERR_COMPLETE_ALREADY: + case NFS4ERR_CONN_NOT_BOUND_TO_SESSION: + case NFS4ERR_DELEG_ALREADY_WANTED: + case NFS4ERR_BACK_CHAN_BUSY: + case NFS4ERR_LAYOUTTRYLATER: + case NFS4ERR_LAYOUTUNAVAILABLE: + case NFS4ERR_NOMATCHING_LAYOUT: + case NFS4ERR_RECALLCONFLICT: + case NFS4ERR_UNKNOWN_LAYOUTTYPE: + case NFS4ERR_SEQ_MISORDERED: + case NFS4ERR_SEQUENCE_POS: + case NFS4ERR_REQ_TOO_BIG: + case NFS4ERR_REP_TOO_BIG: + case NFS4ERR_REP_TOO_BIG_TO_CACHE: + case NFS4ERR_RETRY_UNCACHED_REP: + case NFS4ERR_UNSAFE_COMPOUND: + case NFS4ERR_TOO_MANY_OPS: + case NFS4ERR_OP_NOT_IN_SESSION: + case NFS4ERR_HASH_ALG_UNSUPP: + case NFS4ERR_CLIENTID_BUSY: + case NFS4ERR_PNFS_IO_HOLE: + case NFS4ERR_SEQ_FALSE_RETRY: + case NFS4ERR_BAD_HIGH_SLOT: + case NFS4ERR_DEADSESSION: + case NFS4ERR_ENCR_ALG_UNSUPP: + case NFS4ERR_PNFS_NO_LAYOUT: + case NFS4ERR_NOT_ONLY_OP: + case NFS4ERR_WRONG_CRED: + case NFS4ERR_WRONG_TYPE: + case NFS4ERR_DIRDELEG_UNAVAIL: + case NFS4ERR_REJECT_DELEG: + case NFS4ERR_RETURNCONFLICT: + case NFS4ERR_DELEG_REVOKED: + case NFS4ERR_PARTNER_NOTSUPP: + case NFS4ERR_PARTNER_NO_AUTH: + case NFS4ERR_UNION_NOTSUPP: + case NFS4ERR_OFFLOAD_DENIED: + case NFS4ERR_WRONG_LFS: + case NFS4ERR_BADLABEL: + case NFS4ERR_OFFLOAD_NO_REQS: + case NFS4ERR_NOXATTR: + case NFS4ERR_XATTR2BIG: + break; + default: + return false; + } + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_attrlist4(struct xdr_stream *xdr, attrlist4 *ptr) +{ + return xdrgen_decode_opaque(xdr, ptr, 0); +} + static bool __maybe_unused xdrgen_decode_bitmap4(struct xdr_stream *xdr, bitmap4 *ptr) { @@ -30,6 +178,24 @@ xdrgen_decode_bitmap4(struct xdr_stream *xdr, bitmap4 *ptr) return true; } +static bool __maybe_unused +xdrgen_decode_verifier4(struct xdr_stream *xdr, verifier4 *ptr) +{ + return xdr_stream_decode_opaque_fixed(xdr, ptr, NFS4_VERIFIER_SIZE) == 0; +} + +static bool __maybe_unused +xdrgen_decode_nfs_cookie4(struct xdr_stream *xdr, nfs_cookie4 *ptr) +{ + return xdrgen_decode_uint64_t(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_nfs_fh4(struct xdr_stream *xdr, nfs_fh4 *ptr) +{ + return xdrgen_decode_opaque(xdr, ptr, NFS4_FHSIZE); +} + static bool __maybe_unused xdrgen_decode_utf8string(struct xdr_stream *xdr, utf8string *ptr) { @@ -54,6 +220,29 @@ xdrgen_decode_utf8str_mixed(struct xdr_stream *xdr, utf8str_mixed *ptr) return xdrgen_decode_utf8string(xdr, ptr); } +static bool __maybe_unused +xdrgen_decode_component4(struct xdr_stream *xdr, component4 *ptr) +{ + return xdrgen_decode_utf8str_cs(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_linktext4(struct xdr_stream *xdr, linktext4 *ptr) +{ + return xdrgen_decode_utf8str_cs(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_pathname4(struct xdr_stream *xdr, pathname4 *ptr) +{ + if (xdr_stream_decode_u32(xdr, &ptr->count) < 0) + return false; + for (u32 i = 0; i < ptr->count; i++) + if (!xdrgen_decode_component4(xdr, &ptr->element[i])) + return false; + return true; +} + static bool __maybe_unused xdrgen_decode_nfstime4(struct xdr_stream *xdr, struct nfstime4 *ptr) { @@ -64,6 +253,26 @@ xdrgen_decode_nfstime4(struct xdr_stream *xdr, struct nfstime4 *ptr) return true; } +static bool __maybe_unused +xdrgen_decode_fattr4(struct xdr_stream *xdr, struct fattr4 *ptr) +{ + if (!xdrgen_decode_bitmap4(xdr, &ptr->attrmask)) + return false; + if (!xdrgen_decode_attrlist4(xdr, &ptr->attr_vals)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_stateid4(struct xdr_stream *xdr, struct stateid4 *ptr) +{ + if (!xdrgen_decode_uint32_t(xdr, &ptr->seqid)) + return false; + if (xdr_stream_decode_opaque_fixed(xdr, ptr->other, 12) < 0) + return false; + return true; +} + static bool __maybe_unused xdrgen_decode_fattr4_offline(struct xdr_stream *xdr, fattr4_offline *ptr) { @@ -366,9 +575,160 @@ xdrgen_decode_fattr4_posix_access_acl(struct xdr_stream *xdr, fattr4_posix_acces */ static bool __maybe_unused -xdrgen_encode_int64_t(struct xdr_stream *xdr, const int64_t value) +xdrgen_decode_notify_type4(struct xdr_stream *xdr, notify_type4 *ptr) { - return xdrgen_encode_hyper(xdr, value); + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case NOTIFY4_CHANGE_CHILD_ATTRS: + case NOTIFY4_CHANGE_DIR_ATTRS: + case NOTIFY4_REMOVE_ENTRY: + case NOTIFY4_ADD_ENTRY: + case NOTIFY4_RENAME_ENTRY: + case NOTIFY4_CHANGE_COOKIE_VERIFIER: + break; + default: + return false; + } + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notify_entry4(struct xdr_stream *xdr, struct notify_entry4 *ptr) +{ + if (!xdrgen_decode_component4(xdr, &ptr->ne_file)) + return false; + if (!xdrgen_decode_fattr4(xdr, &ptr->ne_attrs)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_prev_entry4(struct xdr_stream *xdr, struct prev_entry4 *ptr) +{ + if (!xdrgen_decode_notify_entry4(xdr, &ptr->pe_prev_entry)) + return false; + if (!xdrgen_decode_nfs_cookie4(xdr, &ptr->pe_prev_entry_cookie)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notify_remove4(struct xdr_stream *xdr, struct notify_remove4 *ptr) +{ + if (!xdrgen_decode_notify_entry4(xdr, &ptr->nrm_old_entry)) + return false; + if (!xdrgen_decode_nfs_cookie4(xdr, &ptr->nrm_old_entry_cookie)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notify_add4(struct xdr_stream *xdr, struct notify_add4 *ptr) +{ + if (xdr_stream_decode_u32(xdr, &ptr->nad_old_entry.count) < 0) + return false; + if (ptr->nad_old_entry.count > 1) + return false; + for (u32 i = 0; i < ptr->nad_old_entry.count; i++) + if (!xdrgen_decode_notify_remove4(xdr, &ptr->nad_old_entry.element[i])) + return false; + if (!xdrgen_decode_notify_entry4(xdr, &ptr->nad_new_entry)) + return false; + if (xdr_stream_decode_u32(xdr, &ptr->nad_new_entry_cookie.count) < 0) + return false; + if (ptr->nad_new_entry_cookie.count > 1) + return false; + for (u32 i = 0; i < ptr->nad_new_entry_cookie.count; i++) + if (!xdrgen_decode_nfs_cookie4(xdr, &ptr->nad_new_entry_cookie.element[i])) + return false; + if (xdr_stream_decode_u32(xdr, &ptr->nad_prev_entry.count) < 0) + return false; + if (ptr->nad_prev_entry.count > 1) + return false; + for (u32 i = 0; i < ptr->nad_prev_entry.count; i++) + if (!xdrgen_decode_prev_entry4(xdr, &ptr->nad_prev_entry.element[i])) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->nad_last_entry)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notify_attr4(struct xdr_stream *xdr, struct notify_attr4 *ptr) +{ + if (!xdrgen_decode_notify_entry4(xdr, &ptr->na_changed_entry)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notify_rename4(struct xdr_stream *xdr, struct notify_rename4 *ptr) +{ + if (!xdrgen_decode_notify_remove4(xdr, &ptr->nrn_old_entry)) + return false; + if (!xdrgen_decode_notify_add4(xdr, &ptr->nrn_new_entry)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notify_verifier4(struct xdr_stream *xdr, struct notify_verifier4 *ptr) +{ + if (!xdrgen_decode_verifier4(xdr, &ptr->nv_old_cookieverf)) + return false; + if (!xdrgen_decode_verifier4(xdr, &ptr->nv_new_cookieverf)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notifylist4(struct xdr_stream *xdr, notifylist4 *ptr) +{ + return xdrgen_decode_opaque(xdr, ptr, 0); +} + +static bool __maybe_unused +xdrgen_decode_notify4(struct xdr_stream *xdr, struct notify4 *ptr) +{ + if (!xdrgen_decode_bitmap4(xdr, &ptr->notify_mask)) + return false; + if (!xdrgen_decode_notifylist4(xdr, &ptr->notify_vals)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_CB_NOTIFY4args(struct xdr_stream *xdr, struct CB_NOTIFY4args *ptr) +{ + if (!xdrgen_decode_stateid4(xdr, &ptr->cna_stateid)) + return false; + if (!xdrgen_decode_nfs_fh4(xdr, &ptr->cna_fh)) + return false; + if (xdr_stream_decode_u32(xdr, &ptr->cna_changes.count) < 0) + return false; + for (u32 i = 0; i < ptr->cna_changes.count; i++) + if (!xdrgen_decode_notify4(xdr, &ptr->cna_changes.element[i])) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_CB_NOTIFY4res(struct xdr_stream *xdr, struct CB_NOTIFY4res *ptr) +{ + if (!xdrgen_decode_nfsstat4(xdr, &ptr->cnr_status)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_int32_t(struct xdr_stream *xdr, const int32_t value) +{ + return xdrgen_encode_int(xdr, value); } static bool __maybe_unused @@ -377,6 +737,30 @@ xdrgen_encode_uint32_t(struct xdr_stream *xdr, const uint32_t value) return xdrgen_encode_unsigned_int(xdr, value); } +static bool __maybe_unused +xdrgen_encode_int64_t(struct xdr_stream *xdr, const int64_t value) +{ + return xdrgen_encode_hyper(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_uint64_t(struct xdr_stream *xdr, const uint64_t value) +{ + return xdrgen_encode_unsigned_hyper(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_nfsstat4(struct xdr_stream *xdr, nfsstat4 value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_attrlist4(struct xdr_stream *xdr, const attrlist4 value) +{ + return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; +} + static bool __maybe_unused xdrgen_encode_bitmap4(struct xdr_stream *xdr, const bitmap4 value) { @@ -388,6 +772,24 @@ xdrgen_encode_bitmap4(struct xdr_stream *xdr, const bitmap4 value) return true; } +static bool __maybe_unused +xdrgen_encode_verifier4(struct xdr_stream *xdr, const verifier4 value) +{ + return xdr_stream_encode_opaque_fixed(xdr, value, NFS4_VERIFIER_SIZE) >= 0; +} + +static bool __maybe_unused +xdrgen_encode_nfs_cookie4(struct xdr_stream *xdr, const nfs_cookie4 value) +{ + return xdrgen_encode_uint64_t(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_nfs_fh4(struct xdr_stream *xdr, const nfs_fh4 value) +{ + return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; +} + static bool __maybe_unused xdrgen_encode_utf8string(struct xdr_stream *xdr, const utf8string value) { @@ -412,6 +814,29 @@ xdrgen_encode_utf8str_mixed(struct xdr_stream *xdr, const utf8str_mixed value) return xdrgen_encode_utf8string(xdr, value); } +static bool __maybe_unused +xdrgen_encode_component4(struct xdr_stream *xdr, const component4 value) +{ + return xdrgen_encode_utf8str_cs(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_linktext4(struct xdr_stream *xdr, const linktext4 value) +{ + return xdrgen_encode_utf8str_cs(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_pathname4(struct xdr_stream *xdr, const pathname4 value) +{ + if (xdr_stream_encode_u32(xdr, value.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value.count; i++) + if (!xdrgen_encode_component4(xdr, value.element[i])) + return false; + return true; +} + static bool __maybe_unused xdrgen_encode_nfstime4(struct xdr_stream *xdr, const struct nfstime4 *value) { @@ -422,6 +847,26 @@ xdrgen_encode_nfstime4(struct xdr_stream *xdr, const struct nfstime4 *value) return true; } +static bool __maybe_unused +xdrgen_encode_fattr4(struct xdr_stream *xdr, const struct fattr4 *value) +{ + if (!xdrgen_encode_bitmap4(xdr, value->attrmask)) + return false; + if (!xdrgen_encode_attrlist4(xdr, value->attr_vals)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_stateid4(struct xdr_stream *xdr, const struct stateid4 *value) +{ + if (!xdrgen_encode_uint32_t(xdr, value->seqid)) + return false; + if (xdr_stream_encode_opaque_fixed(xdr, value->other, 12) < 0) + return false; + return true; +} + static bool __maybe_unused xdrgen_encode_fattr4_offline(struct xdr_stream *xdr, const fattr4_offline value) { @@ -567,3 +1012,137 @@ xdrgen_encode_fattr4_posix_access_acl(struct xdr_stream *xdr, const fattr4_posix return false; return true; } + +static bool __maybe_unused +xdrgen_encode_notify_type4(struct xdr_stream *xdr, notify_type4 value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_notify_entry4(struct xdr_stream *xdr, const struct notify_entry4 *value) +{ + if (!xdrgen_encode_component4(xdr, value->ne_file)) + return false; + if (!xdrgen_encode_fattr4(xdr, &value->ne_attrs)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_prev_entry4(struct xdr_stream *xdr, const struct prev_entry4 *value) +{ + if (!xdrgen_encode_notify_entry4(xdr, &value->pe_prev_entry)) + return false; + if (!xdrgen_encode_nfs_cookie4(xdr, value->pe_prev_entry_cookie)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_notify_remove4(struct xdr_stream *xdr, const struct notify_remove4 *value) +{ + if (!xdrgen_encode_notify_entry4(xdr, &value->nrm_old_entry)) + return false; + if (!xdrgen_encode_nfs_cookie4(xdr, value->nrm_old_entry_cookie)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_notify_add4(struct xdr_stream *xdr, const struct notify_add4 *value) +{ + if (value->nad_old_entry.count > 1) + return false; + if (xdr_stream_encode_u32(xdr, value->nad_old_entry.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value->nad_old_entry.count; i++) + if (!xdrgen_encode_notify_remove4(xdr, &value->nad_old_entry.element[i])) + return false; + if (!xdrgen_encode_notify_entry4(xdr, &value->nad_new_entry)) + return false; + if (value->nad_new_entry_cookie.count > 1) + return false; + if (xdr_stream_encode_u32(xdr, value->nad_new_entry_cookie.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value->nad_new_entry_cookie.count; i++) + if (!xdrgen_encode_nfs_cookie4(xdr, value->nad_new_entry_cookie.element[i])) + return false; + if (value->nad_prev_entry.count > 1) + return false; + if (xdr_stream_encode_u32(xdr, value->nad_prev_entry.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value->nad_prev_entry.count; i++) + if (!xdrgen_encode_prev_entry4(xdr, &value->nad_prev_entry.element[i])) + return false; + if (!xdrgen_encode_bool(xdr, value->nad_last_entry)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_notify_attr4(struct xdr_stream *xdr, const struct notify_attr4 *value) +{ + if (!xdrgen_encode_notify_entry4(xdr, &value->na_changed_entry)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_notify_rename4(struct xdr_stream *xdr, const struct notify_rename4 *value) +{ + if (!xdrgen_encode_notify_remove4(xdr, &value->nrn_old_entry)) + return false; + if (!xdrgen_encode_notify_add4(xdr, &value->nrn_new_entry)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_notify_verifier4(struct xdr_stream *xdr, const struct notify_verifier4 *value) +{ + if (!xdrgen_encode_verifier4(xdr, value->nv_old_cookieverf)) + return false; + if (!xdrgen_encode_verifier4(xdr, value->nv_new_cookieverf)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_notifylist4(struct xdr_stream *xdr, const notifylist4 value) +{ + return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; +} + +static bool __maybe_unused +xdrgen_encode_notify4(struct xdr_stream *xdr, const struct notify4 *value) +{ + if (!xdrgen_encode_bitmap4(xdr, value->notify_mask)) + return false; + if (!xdrgen_encode_notifylist4(xdr, value->notify_vals)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_CB_NOTIFY4args(struct xdr_stream *xdr, const struct CB_NOTIFY4args *value) +{ + if (!xdrgen_encode_stateid4(xdr, &value->cna_stateid)) + return false; + if (!xdrgen_encode_nfs_fh4(xdr, value->cna_fh)) + return false; + if (xdr_stream_encode_u32(xdr, value->cna_changes.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value->cna_changes.count; i++) + if (!xdrgen_encode_notify4(xdr, &value->cna_changes.element[i])) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_CB_NOTIFY4res(struct xdr_stream *xdr, const struct CB_NOTIFY4res *value) +{ + if (!xdrgen_encode_nfsstat4(xdr, value->cnr_status)) + return false; + return true; +} diff --git a/fs/nfsd/nfs4xdr_gen.h b/fs/nfsd/nfs4xdr_gen.h index 1c487f1a11ab2f..d7725bd5648ae2 100644 --- a/fs/nfsd/nfs4xdr_gen.h +++ b/fs/nfsd/nfs4xdr_gen.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Generated by xdrgen. Manual edits will be lost. */ /* XDR specification file: ../../Documentation/sunrpc/xdr/nfs4_1.x */ -/* XDR specification modification time: Thu Jan 8 23:12:07 2026 */ +/* XDR specification modification time: Tue Jun 30 11:57:21 2026 */ #ifndef _LINUX_XDRGEN_NFS4_1_DECL_H #define _LINUX_XDRGEN_NFS4_1_DECL_H @@ -21,10 +21,13 @@ bool xdrgen_encode_fattr4_time_deleg_access(struct xdr_stream *xdr, const fattr4 bool xdrgen_decode_fattr4_time_deleg_modify(struct xdr_stream *xdr, fattr4_time_deleg_modify *ptr); bool xdrgen_encode_fattr4_time_deleg_modify(struct xdr_stream *xdr, const fattr4_time_deleg_modify *value); + bool xdrgen_decode_aclmodel4(struct xdr_stream *xdr, aclmodel4 *ptr); bool xdrgen_encode_aclmodel4(struct xdr_stream *xdr, aclmodel4 value); + bool xdrgen_decode_aclscope4(struct xdr_stream *xdr, aclscope4 *ptr); bool xdrgen_encode_aclscope4(struct xdr_stream *xdr, aclscope4 value); + bool xdrgen_decode_posixacetag4(struct xdr_stream *xdr, posixacetag4 *ptr); bool xdrgen_encode_posixacetag4(struct xdr_stream *xdr, posixacetag4 value); diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index 11bce03b903162..d33d49b61e6afd 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -332,13 +332,14 @@ void nfsd_lockd_shutdown(void); #define nfserr_noxattr cpu_to_be32(NFS4ERR_NOXATTR) /* - * Error codes for internal use. We use enum to choose numbers that are - * not already assigned, then covert to be32 resulting in a number that - * cannot conflict with any existing be32 nfserr value. + * Error codes for internal use. These are based at an impossible + * nfsstat4 value so that, once converted to be32, they cannot conflict + * with any value defined by the protocol (compare the nlm__int__* codes + * in fs/lockd/lockd.h). */ enum { /* end-of-file indicator in readdir */ - NFSERR_EOF = NFS4ERR_FIRST_FREE, + NFSERR_EOF = 30000, #define nfserr_eof cpu_to_be32(NFSERR_EOF) /* replay detected */ diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index 33953d38314e9b..171e8fdbafb601 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -1677,6 +1677,7 @@ TRACE_EVENT(nfsd_cb_setup_err, { OP_CB_RECALL, "CB_RECALL" }, \ { OP_CB_LAYOUTRECALL, "CB_LAYOUTRECALL" }, \ { OP_CB_RECALL_ANY, "CB_RECALL_ANY" }, \ + { OP_CB_NOTIFY, "CB_NOTIFY" }, \ { OP_CB_NOTIFY_LOCK, "CB_NOTIFY_LOCK" }, \ { OP_CB_OFFLOAD, "CB_OFFLOAD" }) diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c index a9d8aa65416fd0..1d00bce21c3773 100644 --- a/fs/nilfs2/page.c +++ b/fs/nilfs2/page.c @@ -243,6 +243,7 @@ static void nilfs_copy_folio(struct folio *dst, struct folio *src, int nilfs_copy_dirty_pages(struct address_space *dmap, struct address_space *smap) { + struct inode *smap_inode = smap->host; struct folio_batch fbatch; unsigned int i; pgoff_t index = 0; @@ -258,8 +259,19 @@ repeat: struct folio *folio = fbatch.folios[i], *dfolio; folio_lock(folio); - if (unlikely(!folio_test_dirty(folio))) - NILFS_FOLIO_BUG(folio, "inconsistent dirty state"); + if (unlikely(!folio_test_dirty(folio))) { + if (WARN_ONCE(!sb_rdonly(smap_inode->i_sb), + "inconsistent dirty state\n")) + goto unlock_folio; + + /* + * If the filesystem has been forced to read-only + * due to metadata corruption. + */ + folio_unlock(folio); + err = -EROFS; + break; + } dfolio = filemap_grab_folio(dmap, folio->index); if (IS_ERR(dfolio)) { @@ -277,6 +289,7 @@ repeat: folio_unlock(dfolio); folio_put(dfolio); +unlock_folio: folio_unlock(folio); } folio_batch_release(&fbatch); diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c index 9332f5ac608334..2189267894d2c8 100644 --- a/fs/nilfs2/segment.c +++ b/fs/nilfs2/segment.c @@ -2561,6 +2561,10 @@ int nilfs_clean_segments(struct super_block *sb, struct nilfs_argv *argv, break; nilfs_warn(sb, "error %d cleaning segments", err); + + if (unlikely(err == -EROFS)) + goto out_unlock; + set_current_state(TASK_INTERRUPTIBLE); schedule_timeout(sci->sc_interval); } diff --git a/fs/nilfs2/the_nilfs.c b/fs/nilfs2/the_nilfs.c index 7b23e373a106ed..f3805e7aabeb5e 100644 --- a/fs/nilfs2/the_nilfs.c +++ b/fs/nilfs2/the_nilfs.c @@ -461,6 +461,12 @@ static int nilfs_store_disk_layout(struct the_nilfs *nilfs, nilfs->ns_inode_size); return -EINVAL; } + if (NILFS_SR_BYTES(nilfs->ns_inode_size) > nilfs->ns_blocksize) { + nilfs_err(nilfs->ns_sb, + "too large inode size for super root: %d bytes", + nilfs->ns_inode_size); + return -EINVAL; + } nilfs->ns_first_ino = le32_to_cpu(sbp->s_first_ino); if (nilfs->ns_first_ino < NILFS_USER_INO) { diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c index b05b6d3abb870e..a208a7ec1692a2 100644 --- a/fs/notify/fanotify/fanotify.c +++ b/fs/notify/fanotify/fanotify.c @@ -599,6 +599,7 @@ static struct fanotify_event *fanotify_alloc_perm_event(const void *data, pevent->hdr.pad = 0; pevent->hdr.len = 0; pevent->state = FAN_EVENT_INIT; + pevent->watchdog_cnt = 0; pevent->path = *path; pevent->pos = range ? range->pos : FANOTIFY_NO_RANGE; pevent->count = range ? range->count : 0; diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index 7278286f5a6d96..a32c6634d59278 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -112,7 +112,12 @@ static DECLARE_DELAYED_WORK(perm_group_work, perm_group_watchdog); static void perm_group_watchdog_schedule(void) { - schedule_delayed_work(&perm_group_work, secs_to_jiffies(perm_group_timeout)); + int timeout = READ_ONCE(perm_group_timeout); + + if (!timeout) + return; + + schedule_delayed_work(&perm_group_work, secs_to_jiffies(timeout)); } static void perm_group_watchdog(struct work_struct *work) @@ -1142,11 +1147,13 @@ static long fanotify_ioctl(struct file *file, unsigned int cmd, unsigned long ar { struct fsnotify_group *group; struct fsnotify_event *fsn_event; + unsigned int info_mode; void __user *p; int ret = -ENOTTY; size_t send_len = 0; group = file->private_data; + info_mode = FAN_GROUP_FLAG(group, FANOTIFY_INFO_MODES); p = (void __user *) arg; @@ -1154,7 +1161,8 @@ static long fanotify_ioctl(struct file *file, unsigned int cmd, unsigned long ar case FIONREAD: spin_lock(&group->notification_lock); list_for_each_entry(fsn_event, &group->notification_list, list) - send_len += FAN_EVENT_METADATA_LEN; + send_len += fanotify_event_len(info_mode, + FANOTIFY_E(fsn_event)); spin_unlock(&group->notification_lock); ret = put_user(send_len, (int __user *) p); break; diff --git a/fs/ntfs/attrib.c b/fs/ntfs/attrib.c index 0c147b99d4eea2..b4c6137c037b6e 100644 --- a/fs/ntfs/attrib.c +++ b/fs/ntfs/attrib.c @@ -3553,8 +3553,13 @@ int ntfs_attr_record_move_away(struct ntfs_attr_search_ctx *ctx, int extra) unmap_mft_record(ni); err = ntfs_attr_record_move_to(ctx, ni); - if (err) + if (err) { ntfs_error(sb, "Couldn't move attribute to MFT record"); + if (ntfs_mft_record_free(base_ni->vol, ni)) + ntfs_error(sb, "Couldn't free empty MFT record"); + else + ntfs_inode_close(ni); + } return err; } diff --git a/fs/ntfs/compress.c b/fs/ntfs/compress.c index 76bd806b41edd9..4ea1a6bfb3eb72 100644 --- a/fs/ntfs/compress.c +++ b/fs/ntfs/compress.c @@ -96,26 +96,6 @@ void free_compression_buffers(void) mutex_unlock(&ntfs_cb_lock); } -/* - * zero_partial_compressed_page - zero out of bounds compressed page region - * @page: page to zero - * @initialized_size: initialized size of the attribute - */ -static void zero_partial_compressed_page(struct page *page, - const s64 initialized_size) -{ - u8 *kp = page_address(page); - unsigned int kp_ofs; - - ntfs_debug("Zeroing page region outside initialized size."); - if (((s64)page->__folio_index << PAGE_SHIFT) >= initialized_size) { - clear_page(kp); - return; - } - kp_ofs = initialized_size & ~PAGE_MASK; - memset(kp + kp_ofs, 0, PAGE_SIZE - kp_ofs); -} - /* * handle_bounds_compressed_page - test for&handle out of bounds compressed page * @page: page to check and handle @@ -125,9 +105,21 @@ static void zero_partial_compressed_page(struct page *page, static inline void handle_bounds_compressed_page(struct page *page, const loff_t i_size, const s64 initialized_size) { - if ((page->__folio_index >= (initialized_size >> PAGE_SHIFT)) && - (initialized_size < i_size)) - zero_partial_compressed_page(page, initialized_size); + loff_t pos = page_offset(page); + + if ((pos + PAGE_SIZE > initialized_size) && + (initialized_size < i_size)) { + size_t offset; + + ntfs_debug("Zeroing page region outside initialized size."); + if (pos >= initialized_size) + offset = 0; + else + offset = offset_in_page(initialized_size); + zero_user_segment(page, offset, PAGE_SIZE); + } else { + flush_dcache_page(page); + } } /* @@ -185,6 +177,7 @@ static int ntfs_decompress(struct page *dest_pages[], int completed_pages[], /* Variables for uncompressed data / destination. */ struct page *dp; /* Current destination page being worked on. */ + u8 *dp_kaddr; /* Local kmap for the current destination page. */ u8 *dp_addr; /* Current pointer into dp. */ u8 *dp_sb_start; /* Start of current sub-block in dp. */ u8 *dp_sb_end; /* End of current sb in dp (dp_sb_start + NTFS_SB_SIZE). */ @@ -199,6 +192,7 @@ static int ntfs_decompress(struct page *dest_pages[], int completed_pages[], /* Default error code. */ int err = -EOVERFLOW; + dp_kaddr = NULL; ntfs_debug("Entering, cb_size = 0x%x.", cb_size); do_next_sb: ntfs_debug("Beginning sub-block at offset = 0x%zx in the cb.", @@ -231,8 +225,6 @@ return_error: */ handle_bounds_compressed_page(dp, i_size, initialized_size); - flush_dcache_page(dp); - kunmap_local(page_address(dp)); SetPageUptodate(dp); unlock_page(dp); if (di == xpage) @@ -278,7 +270,8 @@ return_error: } /* We have a valid destination page. Setup the destination pointers. */ - dp_addr = (u8 *)page_address(dp) + do_sb_start; + dp_kaddr = kmap_local_page(dp); + dp_addr = dp_kaddr + do_sb_start; /* Now, we are ready to process the current sub-block (sb). */ if (!(le16_to_cpup((__le16 *)cb) & NTFS_SB_IS_COMPRESSED)) { @@ -299,6 +292,8 @@ return_error: /* Advance destination position to next sub-block. */ *dest_ofs += NTFS_SB_SIZE; *dest_ofs &= ~PAGE_MASK; + kunmap_local(dp_kaddr); + dp_kaddr = NULL; if (!(*dest_ofs)) { finalize_page: /* @@ -333,6 +328,8 @@ do_next_tag: } /* We have finished the current sub-block. */ *dest_ofs &= ~PAGE_MASK; + kunmap_local(dp_kaddr); + dp_kaddr = NULL; if (!(*dest_ofs)) goto finalize_page; goto do_next_sb; @@ -352,7 +349,7 @@ do_next_tag: u8 *dp_back_addr; /* Check if we are done / still in range. */ - if (cb >= cb_sb_end || dp_addr > dp_sb_end) + if (cb >= cb_sb_end || dp_addr >= dp_sb_end) break; /* Determine token type and parse appropriately.*/ @@ -438,6 +435,8 @@ do_next_tag: goto do_next_tag; return_overflow: + if (dp_kaddr) + kunmap_local(dp_kaddr); ntfs_error(NULL, "Failed. Returning -EOVERFLOW."); goto return_error; } @@ -465,14 +464,14 @@ int ntfs_read_compressed_block(struct folio *folio) struct page *page = &folio->page; loff_t i_size; s64 initialized_size; - struct address_space *mapping = page->mapping; + struct address_space *mapping = folio->mapping; struct ntfs_inode *ni = NTFS_I(mapping->host); struct ntfs_volume *vol = ni->vol; struct super_block *sb = vol->sb; struct runlist_element *rl; unsigned long flags; u8 *cb, *cb_pos, *cb_end; - unsigned long offset, index = page->__folio_index; + unsigned long offset, index = folio->index; u32 cb_size = ni->itype.compressed.block_size; u64 cb_size_mask = cb_size - 1UL; s64 vcn; @@ -566,7 +565,6 @@ int ntfs_read_compressed_block(struct folio *folio) * least wasting our time. */ if (!PageDirty(page) && (!PageUptodate(page))) { - kmap_local_page(page); continue; } unlock_page(page); @@ -652,8 +650,7 @@ lock_retry_remap: } lock_page(lpage); - memcpy(cb_pos, page_address(lpage) + page_ofs, - vol->cluster_size); + memcpy_from_page(cb_pos, lpage, page_ofs, vol->cluster_size); unlock_page(lpage); put_page(lpage); cb_pos += vol->cluster_size; @@ -692,14 +689,7 @@ lock_retry_remap: for (; cur_page < cb_max_page; cur_page++) { page = pages[cur_page]; if (page) { - if (likely(!cur_ofs)) - clear_page(page_address(page)); - else - memset(page_address(page) + cur_ofs, 0, - PAGE_SIZE - - cur_ofs); - flush_dcache_page(page); - kunmap_local(page_address(page)); + memzero_page(page, cur_ofs, PAGE_SIZE - cur_ofs); SetPageUptodate(page); unlock_page(page); if (cur_page == xpage) @@ -717,8 +707,7 @@ lock_retry_remap: if (cb_max_ofs && cb_pos < cb_end) { page = pages[cur_page]; if (page) - memset(page_address(page) + cur_ofs, 0, - cb_max_ofs - cur_ofs); + memzero_page(page, cur_ofs, cb_max_ofs - cur_ofs); /* * No need to update cb_pos at this stage: * cb_pos += cb_max_ofs - cur_ofs; @@ -739,7 +728,7 @@ lock_retry_remap: for (; cur_page < cb_max_page; cur_page++) { page = pages[cur_page]; if (page) - memcpy(page_address(page) + cur_ofs, cb_pos, + memcpy_to_page(page, cur_ofs, cb_pos, PAGE_SIZE - cur_ofs); cb_pos += PAGE_SIZE - cur_ofs; cur_ofs = 0; @@ -750,7 +739,7 @@ lock_retry_remap: if (cb_max_ofs && cb_pos < cb_end) { page = pages[cur_page]; if (page) - memcpy(page_address(page) + cur_ofs, cb_pos, + memcpy_to_page(page, cur_ofs, cb_pos, cb_max_ofs - cur_ofs); cb_pos += cb_max_ofs - cur_ofs; cur_ofs = cb_max_ofs; @@ -767,8 +756,6 @@ lock_retry_remap: */ handle_bounds_compressed_page(page, i_size, initialized_size); - flush_dcache_page(page); - kunmap_local(page_address(page)); SetPageUptodate(page); unlock_page(page); if (cur2_page == xpage) @@ -804,7 +791,6 @@ lock_retry_remap: page = pages[prev_cur_page]; if (page) { flush_dcache_page(page); - kunmap_local(page_address(page)); unlock_page(page); if (prev_cur_page != xpage) put_page(page); @@ -822,14 +808,15 @@ lock_retry_remap: for (cur_page = 0; cur_page < max_page; cur_page++) { page = pages[cur_page]; if (page) { + folio = page_folio(page); + ntfs_error(vol->sb, "Still have pages left! Terminating them with extreme prejudice. Inode 0x%llx, page index 0x%lx.", - ni->mft_no, page->__folio_index); - flush_dcache_page(page); - kunmap_local(page_address(page)); - unlock_page(page); + ni->mft_no, folio->index); + flush_dcache_folio(folio); + folio_unlock(folio); if (cur_page != xpage) - put_page(page); + folio_put(folio); pages[cur_page] = NULL; } } @@ -864,7 +851,6 @@ err_out: page = pages[i]; if (page) { flush_dcache_page(page); - kunmap_local(page_address(page)); unlock_page(page); if (i != xpage) put_page(page); @@ -1316,7 +1302,6 @@ static int ntfs_write_cb(struct ntfs_inode *ni, loff_t pos, struct page **pages, } pages_disk[i] = pg; lock_page(pg); - kmap_local_page(pg); } outbuf = vmap(pages_disk, pages_count, VM_MAP, PAGE_KERNEL); @@ -1451,7 +1436,6 @@ out: for (i = 0; i < pages_count; i++) { pg = pages_disk[i]; if (pg) { - kunmap_local(page_address(pg)); unlock_page(pg); put_page(pg); } diff --git a/fs/ntfs/ea.c b/fs/ntfs/ea.c index 0cd192752b7cd7..25ff159dbfd38b 100644 --- a/fs/ntfs/ea.c +++ b/fs/ntfs/ea.c @@ -196,6 +196,9 @@ static int ntfs_set_ea(struct inode *inode, const char *name, size_t name_len, struct ea_attr *p_ea; u32 ea_info_qsize = 0; char *ea_buf = NULL; + char *new_ea_buf; + char *old_ea_buf = NULL; + struct ea_information old_ea_info; size_t new_ea_size = ALIGN(struct_size(p_ea, ea_name, 1 + name_len + val_size), 4); s64 ea_off, ea_info_size, all_ea_size, ea_size; @@ -249,6 +252,22 @@ create_ea_info: err = -EEXIST; goto out; } + if ((flags & XATTR_REPLACE) && !val_size) { + old_ea_info = *p_ea_info; + old_ea_buf = kvmemdup(ea_buf, all_ea_size, GFP_NOFS); + if (!old_ea_buf) { + err = -ENOMEM; + goto out; + } + } + + /* Check the final $EA size before removing the old entry. */ + if (val_size && + ntfs_attr_size_bounds_check(ni->vol, AT_EA, + ea_info_qsize - ea_size + new_ea_size)) { + err = -EFBIG; + goto out; + } p_ea = (struct ea_attr *)(ea_buf + ea_off); @@ -267,17 +286,39 @@ create_ea_info: ea_info_qsize -= ea_size; p_ea_info->ea_query_length = cpu_to_le32(ea_info_qsize); - err = ntfs_write_ea(ni, AT_EA_INFORMATION, (char *)p_ea_info, 0, - sizeof(struct ea_information), false); - if (err) - goto out; + if ((flags & XATTR_REPLACE) && !val_size && !ea_info_qsize) { + err = ntfs_attr_remove(ni, AT_EA, AT_UNNAMED, 0); + if (err) + goto out; - err = ntfs_write_ea(ni, AT_EA, ea_buf, 0, ea_info_qsize, true); - if (err) + err = ntfs_attr_remove(ni, AT_EA_INFORMATION, AT_UNNAMED, 0); + if (err) { + /* Restore the original $EA if $EA_INFORMATION removal failed. */ + ntfs_attr_add(ni, AT_EA, AT_UNNAMED, 0, old_ea_buf, + all_ea_size); + ea_info_qsize = le32_to_cpu(old_ea_info.ea_query_length); + } goto out; + } if ((flags & XATTR_REPLACE) && !val_size) { - /* Remove xattr. */ + err = ntfs_write_ea(ni, AT_EA, ea_buf, 0, ea_info_qsize, + true); + if (err) { + ntfs_write_ea(ni, AT_EA, old_ea_buf, 0, + all_ea_size, false); + goto out; + } + + err = ntfs_write_ea(ni, AT_EA_INFORMATION, (char *)p_ea_info, + 0, sizeof(struct ea_information), false); + if (err) { + ntfs_write_ea(ni, AT_EA, old_ea_buf, 0, + all_ea_size, false); + ntfs_write_ea(ni, AT_EA_INFORMATION, + (char *)&old_ea_info, 0, + sizeof(old_ea_info), false); + } goto out; } } else { @@ -285,22 +326,30 @@ create_ea_info: err = -ENODATA; goto out; } - } - kvfree(ea_buf); + if (ntfs_attr_size_bounds_check(ni->vol, AT_EA, + ea_info_qsize + new_ea_size)) { + err = -EFBIG; + goto out; + } + } alloc_new_ea: - ea_buf = kzalloc(new_ea_size, GFP_NOFS); - if (!ea_buf) { + new_ea_buf = kvzalloc(ea_info_qsize + new_ea_size, GFP_NOFS); + if (!new_ea_buf) { err = -ENOMEM; goto out; } + if (ea_info_qsize) + memcpy(new_ea_buf, ea_buf, ea_info_qsize); + kvfree(ea_buf); + ea_buf = new_ea_buf; + p_ea = (struct ea_attr *)(ea_buf + ea_info_qsize); /* * EA and REPARSE_POINT compatibility not checked any more, * required by Windows 10, but having both may lead to * problems with earlier versions. */ - p_ea = (struct ea_attr *)ea_buf; memcpy(p_ea->ea_name, name, name_len); p_ea->ea_name_length = name_len; p_ea->ea_name[name_len] = 0; @@ -312,8 +361,7 @@ alloc_new_ea: p_ea_info->ea_length = cpu_to_le16(ea_packed); p_ea_info->ea_query_length = cpu_to_le32(ea_info_qsize + new_ea_size); - if (ea_packed > 0xffff || - ntfs_attr_size_bounds_check(ni->vol, AT_EA, new_ea_size)) { + if (ea_packed > 0xffff) { err = -EFBIG; goto out; } @@ -322,13 +370,13 @@ alloc_new_ea: * no EA or EA_INFORMATION : add them */ if (!ntfs_attr_exist(ni, AT_EA, AT_UNNAMED, 0)) { - err = ntfs_attr_add(ni, AT_EA, AT_UNNAMED, 0, (char *)p_ea, - new_ea_size); + err = ntfs_attr_add(ni, AT_EA, AT_UNNAMED, 0, ea_buf, + ea_info_qsize + new_ea_size); if (err) goto out; } else { - err = ntfs_write_ea(ni, AT_EA, (char *)p_ea, ea_info_qsize, - new_ea_size, false); + err = ntfs_write_ea(ni, AT_EA, ea_buf, 0, + ea_info_qsize + new_ea_size, true); if (err) goto out; } @@ -348,6 +396,7 @@ out: NInoClearHasEA(ni); kvfree(ea_buf); + kvfree(old_ea_buf); kvfree(p_ea_info); return err; @@ -357,37 +406,35 @@ out: * Check for the presence of an EA "$LXDEV" (used by WSL) * and return its value as a device address */ -int ntfs_ea_get_wsl_inode(struct inode *inode, dev_t *rdevp, unsigned int flags) +int ntfs_ea_get_wsl_inode(struct inode *inode, dev_t *rdevp, unsigned int flags, + bool *has_lxmod) { int err; __le32 v; + *has_lxmod = false; + if (!(flags & NTFS_VOL_UID)) { /* Load uid to lxuid EA */ err = ntfs_get_ea(inode, "$LXUID", sizeof("$LXUID") - 1, &v, sizeof(v)); - if (err < 0) - return err; - if (err != sizeof(v)) - return -EIO; - i_uid_write(inode, le32_to_cpu(v)); + if (err == sizeof(v)) + i_uid_write(inode, le32_to_cpu(v)); } if (!(flags & NTFS_VOL_GID)) { /* Load gid to lxgid EA */ err = ntfs_get_ea(inode, "$LXGID", sizeof("$LXGID") - 1, &v, sizeof(v)); - if (err < 0) - return err; - if (err != sizeof(v)) - return -EIO; - i_gid_write(inode, le32_to_cpu(v)); + if (err == sizeof(v)) + i_gid_write(inode, le32_to_cpu(v)); } /* Load mode to lxmod EA */ err = ntfs_get_ea(inode, "$LXMOD", sizeof("$LXMOD") - 1, &v, sizeof(v)); if (err == sizeof(v)) { inode->i_mode = le32_to_cpu(v); + *has_lxmod = true; } else { /* Everyone gets all permissions. */ inode->i_mode |= 0777; @@ -704,6 +751,12 @@ err_out: return err; } +static bool ntfs_is_reserved_lxattr(const char *name) +{ + return !strcmp(name, "$LXUID") || !strcmp(name, "$LXGID") || + !strcmp(name, "$LXMOD") || !strcmp(name, "$LXDEV"); +} + static int ntfs_setxattr(const struct xattr_handler *handler, struct mnt_idmap *idmap, struct dentry *unused, struct inode *inode, const char *name, const void *value, @@ -716,6 +769,9 @@ static int ntfs_setxattr(const struct xattr_handler *handler, if (NVolShutdown(ni->vol)) return -EIO; + if (ntfs_is_reserved_lxattr(name) && !capable(CAP_SYS_ADMIN)) + return -EPERM; + if (!strcmp(name, SYSTEM_DOS_ATTRIB)) { if (sizeof(u8) != size) { err = -EINVAL; @@ -768,8 +824,10 @@ set_fattr: mutex_unlock(&ni->mrec_lock); out: - inode_set_ctime_current(inode); - mark_inode_dirty(inode); + if (!err) { + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + } return err; } diff --git a/fs/ntfs/ea.h b/fs/ntfs/ea.h index 1f63bd55e057b0..acb39c2a6fbc86 100644 --- a/fs/ntfs/ea.h +++ b/fs/ntfs/ea.h @@ -10,7 +10,8 @@ extern const struct xattr_handler *const ntfs_xattr_handlers[]; int ntfs_ea_set_wsl_not_symlink(struct ntfs_inode *ni, mode_t mode, dev_t dev); -int ntfs_ea_get_wsl_inode(struct inode *inode, dev_t *rdevp, unsigned int flags); +int ntfs_ea_get_wsl_inode(struct inode *inode, dev_t *rdevp, unsigned int flags, + bool *has_lxmod); int ntfs_ea_set_wsl_inode(struct inode *inode, dev_t rdev, __le16 *ea_size, unsigned int flags); ssize_t ntfs_listxattr(struct dentry *dentry, char *buffer, size_t size); diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c index 6a7b638e523d91..7c1cf27bcaad83 100644 --- a/fs/ntfs/file.c +++ b/fs/ntfs/file.c @@ -346,14 +346,12 @@ int ntfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, if (ia_valid & ATTR_MODE) flags |= NTFS_EA_MODE; - if (S_ISDIR(vi->i_mode)) - vi->i_mode &= ~vol->dmask; - else - vi->i_mode &= ~vol->fmask; - mutex_lock(&ni->mrec_lock); - ntfs_ea_set_wsl_inode(vi, 0, NULL, flags); + err = ntfs_ea_set_wsl_inode(vi, 0, NULL, flags); mutex_unlock(&ni->mrec_lock); + if (err) + goto out; + } mark_inode_dirty(vi); diff --git a/fs/ntfs/index.c b/fs/ntfs/index.c index faa7ee920a3ad0..8c8eb13859ab60 100644 --- a/fs/ntfs/index.c +++ b/fs/ntfs/index.c @@ -616,6 +616,31 @@ static struct index_root *ntfs_ir_lookup2(struct ntfs_inode *ni, __le16 *name, u return ir; } +static int ntfs_ir_move_to_base(struct ntfs_index_context *icx) +{ + struct ntfs_attr_search_ctx *ctx = NULL; + struct index_root *ir; + bool moved = false; + int ret = 0; + + ir = ntfs_ir_lookup(icx->idx_ni, icx->name, icx->name_len, &ctx); + if (!ir) + return -ENOENT; + + if (ctx->ntfs_ino->mft_no != icx->idx_ni->mft_no) { + ret = ntfs_attr_record_move_to(ctx, icx->idx_ni); + if (!ret) { + moved = true; + ret = ntfs_attrlist_update(icx->idx_ni); + } + } + + ntfs_attr_put_search_ctx(ctx); + if (!ret && moved) + ret = ntfs_inode_free_empty_extents(icx->idx_ni); + return ret; +} + /* * Find a key in the index block. */ @@ -989,6 +1014,7 @@ static s64 ntfs_ibm_pos_to_vcn(struct ntfs_index_context *icx, s64 pos) static int ntfs_ibm_add(struct ntfs_index_context *icx) { u8 bmp[8]; + int ret; ntfs_debug("Entering\n"); @@ -998,10 +1024,11 @@ static int ntfs_ibm_add(struct ntfs_index_context *icx) * AT_BITMAP must be at least 8 bytes. */ memset(bmp, 0, sizeof(bmp)); - if (ntfs_attr_add(icx->idx_ni, AT_BITMAP, icx->name, icx->name_len, - bmp, sizeof(bmp))) { + ret = ntfs_attr_add(icx->idx_ni, AT_BITMAP, icx->name, icx->name_len, + bmp, sizeof(bmp)); + if (ret) { ntfs_error(icx->idx_ni->vol->sb, "Failed to add AT_BITMAP"); - return -EINVAL; + return ret; } return 0; @@ -1074,6 +1101,7 @@ static s64 ntfs_ibm_get_free(struct ntfs_index_context *icx) { u8 *bm; int bit; + int ret; s64 vcn, byte, size; ntfs_debug("Entering\n"); @@ -1081,7 +1109,7 @@ static s64 ntfs_ibm_get_free(struct ntfs_index_context *icx) bm = ntfs_attr_readall(icx->idx_ni, AT_BITMAP, icx->name, icx->name_len, &size); if (!bm) - return (s64)-1; + return -EIO; for (byte = 0; byte < size; byte++) { if (bm[byte] == 255) @@ -1099,10 +1127,12 @@ static s64 ntfs_ibm_get_free(struct ntfs_index_context *icx) out: ntfs_debug("allocated vcn: %lld\n", vcn); - if (ntfs_ibm_set(icx, vcn)) - vcn = (s64)-1; + ret = ntfs_ibm_set(icx, vcn); kvfree(bm); + if (ret) + return ret; + return vcn; } @@ -1112,6 +1142,7 @@ static struct index_block *ntfs_ir_to_ib(struct index_root *ir, s64 ib_vcn) struct index_entry *ie_last; char *ies_start, *ies_end; int i; + u32 ib_cap; ntfs_debug("Entering\n"); @@ -1127,6 +1158,16 @@ static struct index_block *ntfs_ir_to_ib(struct index_root *ir, s64 ib_vcn) * as well, which can never have any data. */ i = (char *)ie_last - ies_start + le16_to_cpu(ie_last->length); + + /* Entries must fit in the allocated index block */ + ib_cap = le32_to_cpu(ib->index.allocated_size) - + le32_to_cpu(ib->index.entries_offset); + if ((u32)i > ib_cap) { + ntfs_error(NULL, "Entries (%d B) exceed IB capacity", i); + kvfree(ib); + return NULL; + } + memcpy(ntfs_ie_get_first(&ib->index), ies_start, i); ib->index.flags = ir->index.flags; @@ -1264,7 +1305,7 @@ static int ntfs_ir_reparent(struct ntfs_index_context *icx) new_ib_vcn = ntfs_ibm_get_free(icx); if (new_ib_vcn < 0) { - ret = -EINVAL; + ret = (int)new_ib_vcn; goto out; } @@ -1335,19 +1376,42 @@ resize_failed: * When there is no space to build a non-resident * index, we may have to move the root to an extent */ - if ((ret == -ENOSPC) && (ctx->al_entry || !ntfs_inode_add_attrlist(icx->idx_ni))) { - ntfs_attr_put_search_ctx(ctx); - ctx = NULL; - ir = ntfs_ir_lookup(icx->idx_ni, icx->name, icx->name_len, &ctx); - if (ir && !ntfs_attr_record_move_away(ctx, ix_root_size - - le32_to_cpu(ctx->attr->data.resident.value_length))) { - if (ntfs_attrlist_update(ctx->base_ntfs_ino ? - ctx->base_ntfs_ino : ctx->ntfs_ino)) + if (ret == -ENOSPC) { + if (!ctx->al_entry) { + ret = ntfs_inode_add_attrlist(icx->idx_ni); + if (ret) goto clear_bmp; + ntfs_attr_put_search_ctx(ctx); ctx = NULL; goto retry; } + + if (ctx->ntfs_ino->mft_no != icx->idx_ni->mft_no) + goto clear_bmp; + + ret = ntfs_attr_record_move_away(ctx, ix_root_size - + le32_to_cpu(ctx->attr->data.resident.value_length)); + if (ret) + goto clear_bmp; + + ret = ntfs_attrlist_update(icx->idx_ni); + if (ret) { + int rollback_ret; + + ntfs_attr_put_search_ctx(ctx); + ctx = NULL; + rollback_ret = ntfs_ir_move_to_base(icx); + if (rollback_ret) + ntfs_error(icx->idx_ni->vol->sb, + "Failed to roll back INDEX_ROOT relocation: %d", + rollback_ret); + goto clear_bmp; + } + + ntfs_attr_put_search_ctx(ctx); + ctx = NULL; + goto retry; } clear_bmp: ntfs_ibm_clear(icx, new_ib_vcn); @@ -1579,7 +1643,7 @@ resplit: median = ntfs_ie_get_median(&ib->index); new_vcn = ntfs_ibm_get_free(icx); if (new_vcn < 0) { - ret = -EINVAL; + ret = (int)new_vcn; goto out; } diff --git a/fs/ntfs/inode.c b/fs/ntfs/inode.c index 7381a18cfadd97..f40f35afcda9a9 100644 --- a/fs/ntfs/inode.c +++ b/fs/ntfs/inode.c @@ -682,6 +682,7 @@ static int ntfs_read_locked_inode(struct inode *vi) unsigned int name_len = 4, flags = 0; int extend_sys = 0; dev_t dev = 0; + bool has_lxmod = false; bool vol_err = true; ntfs_debug("Entering for i_ino 0x%llx.", ni->mft_no); @@ -862,7 +863,7 @@ skip_attr_list_load: err = ntfs_attr_lookup(AT_EA_INFORMATION, NULL, 0, 0, 0, NULL, 0, ctx); if (!err) { NInoSetHasEA(ni); - ntfs_ea_get_wsl_inode(vi, &dev, flags); + ntfs_ea_get_wsl_inode(vi, &dev, flags, &has_lxmod); } if (ni->flags & FILE_ATTR_REPARSE_POINT) { @@ -886,16 +887,18 @@ skip_attr_list_load: if (S_ISDIR(vi->i_mode)) { /* - * Apply the directory permissions mask set in the mount - * options. + * Apply the directory permissions mask set in the mount options + * when no per-file WSL mode is present. */ - vi->i_mode &= ~vol->dmask; + if (!has_lxmod) + vi->i_mode &= ~vol->dmask; /* Things break without this kludge! */ if (vi->i_nlink > 1) set_nlink(vi, 1); } else { - /* Apply the file permissions mask set in the mount options. */ - vi->i_mode &= ~vol->fmask; + /* Apply the file permissions mask when no WSL mode is present. */ + if (!has_lxmod) + vi->i_mode &= ~vol->fmask; } /* @@ -3046,6 +3049,7 @@ int ntfs_inode_add_attrlist(struct ntfs_inode *ni) struct attr_list_entry *ale = NULL; struct mft_record *ni_mrec; u32 attr_al_len; + bool free_empty_extents = true; if (!ni) return -EINVAL; @@ -3145,6 +3149,7 @@ int ntfs_inode_add_attrlist(struct ntfs_inode *ni) ntfs_error(ni->vol->sb, "Couldn't add $ATTRIBUTE_LIST to MFT"); goto rollback; } + free_empty_extents = false; err = ntfs_attrlist_update(ni); if (err < 0) @@ -3164,6 +3169,8 @@ remove_attrlist_record: CASE_SENSITIVE, 0, NULL, 0, ctx)) { if (ntfs_attr_record_rm(ctx)) ntfs_error(ni->vol->sb, "Rollback failed to remove attrlist"); + else + free_empty_extents = true; } else { ntfs_error(ni->vol->sb, "Rollback failed to find attrlist"); } @@ -3202,6 +3209,11 @@ rollback: ni->attr_list_size = 0; NInoClearAttrList(ni); NInoClearAttrListDirty(ni); + ntfs_attr_put_search_ctx(ctx); + ctx = NULL; + if (free_empty_extents && ntfs_inode_free_empty_extents(ni)) + ntfs_error(ni->vol->sb, "Rollback failed to free empty extent"); + goto err_out; put_err_out: ntfs_attr_put_search_ctx(ctx); err_out: @@ -3288,6 +3300,55 @@ out: return err; } +/* + * ntfs_inode_free_empty_extents - free empty extent MFT records + * @ni: base inode whose empty extent records should be freed + * + * The caller must ensure that no on-disk attribute list references an empty + * extent record and must hold @ni->mrec_lock to serialize the extent array. + */ +int ntfs_inode_free_empty_extents(struct ntfs_inode *ni) +{ + int err = 0, i = 0; + + if (!ni || ni->nr_extents < 0) + return -EINVAL; + + mutex_lock(&ni->extent_lock); + while (i < ni->nr_extents) { + struct ntfs_inode *ext_ni = ni->ext.extent_ntfs_inos[i]; + struct mft_record *m; + int ret; + + m = map_mft_record(ext_ni); + if (IS_ERR(m)) { + if (!err) + err = PTR_ERR(m); + i++; + continue; + } + if (le32_to_cpu(m->bytes_in_use) - + le16_to_cpu(m->attrs_offset) != 8) { + unmap_mft_record(ext_ni); + i++; + continue; + } + unmap_mft_record(ext_ni); + + ret = ntfs_mft_record_free(ni->vol, ext_ni); + if (ret) { + if (!err) + err = ret; + i++; + continue; + } + ntfs_inode_close(ext_ni); + /* ntfs_inode_close() removed this entry from the extent array. */ + } + mutex_unlock(&ni->extent_lock); + return err; +} + void ntfs_destroy_ext_inode(struct ntfs_inode *ni) { ntfs_debug("Entering."); @@ -3387,6 +3448,9 @@ int ntfs_inode_free_space(struct ntfs_inode *ni, int size) * Chkdsk complain if $STANDARD_INFORMATION is not in the base MFT * record. * + * $INDEX_ROOT must remain resident, but its attribute record may be moved + * to an extent MFT record when the base record needs room for the list. + * * Also we can't move $ATTRIBUTE_LIST from base MFT_RECORD, so position * search context on first attribute after $STANDARD_INFORMATION and * $ATTRIBUTE_LIST. @@ -3428,9 +3492,6 @@ retry: ctx->attr->type == AT_DATA) goto retry; - if (ctx->attr->type == AT_INDEX_ROOT) - goto retry; - record_size = le32_to_cpu(ctx->attr->length); /* Move away attribute. */ diff --git a/fs/ntfs/inode.h b/fs/ntfs/inode.h index 9aacd5787ffea0..bb48f55b50ddcd 100644 --- a/fs/ntfs/inode.h +++ b/fs/ntfs/inode.h @@ -338,6 +338,7 @@ int ntfs_get_block_mft_record(struct ntfs_inode *mft_ni, struct ntfs_inode *ni); int __ntfs_write_inode(struct inode *vi, int sync); int ntfs_inode_attach_all_extents(struct ntfs_inode *ni); int ntfs_inode_add_attrlist(struct ntfs_inode *ni); +int ntfs_inode_free_empty_extents(struct ntfs_inode *ni); void ntfs_destroy_ext_inode(struct ntfs_inode *ni); int ntfs_inode_free_space(struct ntfs_inode *ni, int size); s64 ntfs_inode_attr_pread(struct inode *vi, s64 pos, s64 count, u8 *buf); diff --git a/fs/ntfs/iomap.c b/fs/ntfs/iomap.c index 52eecf5cb256aa..e6c2515014c353 100644 --- a/fs/ntfs/iomap.c +++ b/fs/ntfs/iomap.c @@ -81,7 +81,7 @@ const struct iomap_write_ops ntfs_iomap_folio_ops = { }; static int ntfs_read_iomap_begin_resident(struct inode *inode, loff_t offset, loff_t length, - unsigned int flags, struct iomap *iomap) + unsigned int flags, struct iomap *iomap, bool keep_mrec_lock) { struct ntfs_inode *base_ni, *ni = NTFS_I(inode); struct ntfs_attr_search_ctx *ctx; @@ -95,6 +95,8 @@ static int ntfs_read_iomap_begin_resident(struct inode *inode, loff_t offset, lo else base_ni = ni; + mutex_lock(&base_ni->mrec_lock); + ctx = ntfs_attr_get_search_ctx(base_ni, NULL); if (!ctx) { err = -ENOMEM; @@ -138,6 +140,13 @@ out: if (ctx) ntfs_attr_put_search_ctx(ctx); + if (!err && keep_mrec_lock && iomap->type == IOMAP_INLINE) { + iomap->private = base_ni; + return 0; + } + + mutex_unlock(&base_ni->mrec_lock); + return err; } @@ -261,24 +270,37 @@ static int ntfs_read_iomap_begin_non_resident(struct inode *inode, loff_t offset static int __ntfs_read_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned int flags, struct iomap *iomap, struct iomap *srcmap, - bool need_unwritten) + bool need_unwritten, bool keep_mrec_lock) { if (NInoNonResident(NTFS_I(inode))) return ntfs_read_iomap_begin_non_resident(inode, offset, length, flags, iomap, need_unwritten); return ntfs_read_iomap_begin_resident(inode, offset, length, - flags, iomap); + flags, iomap, keep_mrec_lock); } static int ntfs_read_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned int flags, struct iomap *iomap, struct iomap *srcmap) { return __ntfs_read_iomap_begin(inode, offset, length, flags, iomap, - srcmap, true); + srcmap, true, true); } +static int ntfs_read_iomap_end(struct inode *inode, loff_t pos, loff_t length, + ssize_t written, unsigned int flags, struct iomap *iomap) +{ + struct ntfs_inode *base_ni = iomap->private; + + if (base_ni) + mutex_unlock(&base_ni->mrec_lock); + return written; +} + +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_read_iomap_next, + ntfs_read_iomap_begin, ntfs_read_iomap_end); + const struct iomap_ops ntfs_read_iomap_ops = { - .iomap_begin = ntfs_read_iomap_begin, + .iomap_next = ntfs_read_iomap_next, }; /* @@ -318,7 +340,7 @@ static int ntfs_seek_iomap_begin(struct inode *inode, loff_t offset, loff_t leng unsigned int flags, struct iomap *iomap, struct iomap *srcmap) { return __ntfs_read_iomap_begin(inode, offset, length, flags, iomap, - srcmap, false); + srcmap, false, false); } static int ntfs_zero_read_iomap_end(struct inode *inode, loff_t pos, loff_t length, @@ -329,13 +351,17 @@ static int ntfs_zero_read_iomap_end(struct inode *inode, loff_t pos, loff_t leng return written; } +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_zero_read_iomap_next, + ntfs_seek_iomap_begin, ntfs_zero_read_iomap_end); + static const struct iomap_ops ntfs_zero_read_iomap_ops = { - .iomap_begin = ntfs_seek_iomap_begin, - .iomap_end = ntfs_zero_read_iomap_end, + .iomap_next = ntfs_zero_read_iomap_next, }; +static DEFINE_IOMAP_ITER_NEXT(ntfs_seek_iomap_next, ntfs_seek_iomap_begin); + const struct iomap_ops ntfs_seek_iomap_ops = { - .iomap_begin = ntfs_seek_iomap_begin, + .iomap_next = ntfs_seek_iomap_next, }; int ntfs_dio_zero_range(struct inode *inode, loff_t offset, loff_t length) @@ -764,9 +790,11 @@ static int ntfs_write_iomap_end(struct inode *inode, loff_t pos, loff_t length, return written; } +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_write_iomap_next, + ntfs_write_iomap_begin, ntfs_write_iomap_end); + const struct iomap_ops ntfs_write_iomap_ops = { - .iomap_begin = ntfs_write_iomap_begin, - .iomap_end = ntfs_write_iomap_end, + .iomap_next = ntfs_write_iomap_next, }; static int ntfs_page_mkwrite_iomap_begin(struct inode *inode, loff_t offset, @@ -777,9 +805,11 @@ static int ntfs_page_mkwrite_iomap_begin(struct inode *inode, loff_t offset, NTFS_IOMAP_FLAGS_MKWRITE); } +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_page_mkwrite_iomap_next, + ntfs_page_mkwrite_iomap_begin, ntfs_write_iomap_end); + const struct iomap_ops ntfs_page_mkwrite_iomap_ops = { - .iomap_begin = ntfs_page_mkwrite_iomap_begin, - .iomap_end = ntfs_write_iomap_end, + .iomap_next = ntfs_page_mkwrite_iomap_next, }; static int ntfs_dio_iomap_begin(struct inode *inode, loff_t offset, @@ -790,9 +820,11 @@ static int ntfs_dio_iomap_begin(struct inode *inode, loff_t offset, NTFS_IOMAP_FLAGS_DIO); } +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_dio_iomap_next, + ntfs_dio_iomap_begin, ntfs_write_iomap_end); + const struct iomap_ops ntfs_dio_iomap_ops = { - .iomap_begin = ntfs_dio_iomap_begin, - .iomap_end = ntfs_write_iomap_end, + .iomap_next = ntfs_dio_iomap_next, }; static ssize_t ntfs_writeback_range(struct iomap_writepage_ctx *wpc, diff --git a/fs/ntfs/mft.c b/fs/ntfs/mft.c index fd20d7abd6f5ec..271a265491280e 100644 --- a/fs/ntfs/mft.c +++ b/fs/ntfs/mft.c @@ -2333,7 +2333,17 @@ mft_rec_already_initialized: * wrong with the previous mft record. */ seq_no = m->sequence_number; - usn = *(__le16 *)((u8 *)m + le16_to_cpu(m->usa_ofs)); + /* + * The mft record still holds unvalidated, MST-protected on-disk + * bytes, so m->usa_ofs is untrusted here. Only preserve the old + * update sequence number if that offset is in bounds; otherwise + * leave usn zero so it is not restored below. + */ + if (!(le16_to_cpu(m->usa_ofs) & 1) && + le16_to_cpu(m->usa_ofs) + sizeof(usn) <= vol->mft_record_size) + usn = *(__le16 *)((u8 *)m + le16_to_cpu(m->usa_ofs)); + else + usn = 0; err = ntfs_mft_record_layout(vol, bit, m); if (unlikely(err)) { ntfs_error(vol->sb, "Failed to layout allocated mft record 0x%llx.", diff --git a/fs/ntfs/namei.c b/fs/ntfs/namei.c index cd403b1d99eed9..96045face63f99 100644 --- a/fs/ntfs/namei.c +++ b/fs/ntfs/namei.c @@ -61,12 +61,12 @@ static int ntfs_check_bad_windows_name(struct ntfs_volume *vol, const __le16 *wc, unsigned int wc_len) { - if (ntfs_check_bad_char(wc, wc_len)) - return -EINVAL; - if (!NVolCheckWindowsNames(vol)) return 0; + if (ntfs_check_bad_char(wc, wc_len)) + return -EINVAL; + /* Check for trailing space or dot. */ if (wc_len > 0 && (wc[wc_len - 1] == cpu_to_le16(' ') || @@ -424,8 +424,6 @@ static struct ntfs_inode *__ntfs_create(struct mnt_idmap *idmap, struct inode *d * directories, also setup the index values to the defaults. */ if (S_ISDIR(mode)) { - mode &= ~vol->dmask; - NInoSetMstProtected(ni); ni->itype.index.block_size = 4096; ni->itype.index.block_size_bits = ntfs_ffs(4096) - 1; @@ -439,8 +437,6 @@ static struct ntfs_inode *__ntfs_create(struct mnt_idmap *idmap, struct inode *d ni->itype.index.vcn_size_bits = vol->sector_size_bits; } - } else { - mode &= ~vol->fmask; } if (IS_RDONLY(vi)) diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c index c759841b74309a..b05e48b28625ae 100644 --- a/fs/ntfs3/fslog.c +++ b/fs/ntfs3/fslog.c @@ -875,6 +875,9 @@ static inline struct RESTART_TABLE *extend_rsttbl(struct RESTART_TABLE *tbl, u32 used = le16_to_cpu(tbl->used); struct RESTART_TABLE *rt; + if (used + add > U16_MAX) + return NULL; + rt = init_rsttbl(esize, used + add); if (!rt) return NULL; @@ -2298,7 +2301,15 @@ static int read_log_rec_buf(struct ntfs_log *log, */ for (;;) { bool usa_error; - u32 tail = log->page_size - off; + u32 tail; + + /* off comes from the on-disk restart area; bound it. */ + if (off > log->page_size) { + err = -EINVAL; + goto out; + } + + tail = log->page_size - off; if (tail >= data_len) tail = data_len; diff --git a/fs/ntfs3/fsntfs.c b/fs/ntfs3/fsntfs.c index bc7469d0a34d40..7c4db816c43d3d 100644 --- a/fs/ntfs3/fsntfs.c +++ b/fs/ntfs3/fsntfs.c @@ -2302,8 +2302,8 @@ int ntfs_reparse_init(struct ntfs_sb_info *sbi) goto out; } - root_r = resident_data(attr); - if (root_r->type != ATTR_ZERO || + root_r = resident_data_ex(attr, sizeof(struct INDEX_ROOT)); + if (!root_r || root_r->type != ATTR_ZERO || root_r->rule != NTFS_COLLATION_TYPE_UINTS) { err = -EINVAL; goto out; @@ -2340,8 +2340,8 @@ int ntfs_objid_init(struct ntfs_sb_info *sbi) goto out; } - root = resident_data(attr); - if (root->type != ATTR_ZERO || + root = resident_data_ex(attr, sizeof(struct INDEX_ROOT)); + if (!root || root->type != ATTR_ZERO || root->rule != NTFS_COLLATION_TYPE_UINTS) { err = -EINVAL; goto out; diff --git a/fs/ntfs3/record.c b/fs/ntfs3/record.c index 32bdb034c2a303..4f12ce15b03b43 100644 --- a/fs/ntfs3/record.c +++ b/fs/ntfs3/record.c @@ -202,7 +202,7 @@ struct ATTRIB *mi_enum_attr(struct ntfs_inode *ni, struct mft_inode *mi, u32 used = le32_to_cpu(rec->used); u32 t32, off, asize, prev_type; u16 t16; - u64 data_size, alloc_size, tot_size; + u64 svcn, evcn, data_size, alloc_size, tot_size; if (!attr) { u32 total = le32_to_cpu(rec->total); @@ -310,10 +310,38 @@ struct ATTRIB *mi_enum_attr(struct ntfs_inode *ni, struct mft_inode *mi, if (t32 && le16_to_cpu(attr->name_off) + t32 > t16) goto out; - /* Check start/end vcn. */ - if (le64_to_cpu(attr->nres.svcn) > le64_to_cpu(attr->nres.evcn) + 1) + /* + * Check start/end vcn. svcn == 0 with evcn == -1 (U64_MAX) is the + * sentinel for an empty non-resident attribute (no allocated + * clusters) and must be accepted: "svcn > evcn + 1" tolerates it, + * since "(u64)-1 + 1" is 0 and "0 > 0" is false. + * + * For a non-empty attribute evcn is a cluster index and must lie + * within the volume (sbi->used.bitmap.nbits, set up in + * ntfs_init_from_boot() before any caller of mi_enum_attr() runs). + * Bounding evcn also prevents a malformed value close to U64_MAX + * from slipping through the near-wrap "evcn + 1" upper bound. + */ + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + if (svcn > evcn + 1) goto out; + if (is_attr_ext(attr)) { + /* sparsed/compressed attribute. */ +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + /* No limits. */ +#else + /* Check evcn fits into 32 bits. */ + if (evcn != U64_MAX && evcn >= (1ull << 32)) + goto out; +#endif + } else { + /* Check out of volume for normal attribute. */ + if (evcn != U64_MAX && evcn >= mi->sbi->used.bitmap.nbits) + goto out; + } + data_size = le64_to_cpu(attr->nres.data_size); if (le64_to_cpu(attr->nres.valid_size) > data_size) goto out; diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c index 3305fe406cb223..4205a212154b81 100644 --- a/fs/ntfs3/super.c +++ b/fs/ntfs3/super.c @@ -65,6 +65,7 @@ #include #include #include +#include #include #include #include @@ -957,7 +958,7 @@ static int ntfs_init_from_boot(struct super_block *sb, u32 sector_size, struct ntfs_sb_info *sbi = sb->s_fs_info; int err; u32 mb, gb, boot_sector_size, sct_per_clst, record_size; - u64 sectors, clusters, mlcn, mlcn2, dev_size0; + u64 sectors, clusters, mlcn, mlcn2, mft_pos, mft2_pos, dev_size0; struct NTFS_BOOT *boot; struct buffer_head *bh; struct MFT_REC *rec; @@ -1026,7 +1027,15 @@ read_boot: mlcn2 = le64_to_cpu(boot->mft2_clst); sectors = le64_to_cpu(boot->sectors_per_volume); - if (mlcn * sct_per_clst >= sectors || mlcn2 * sct_per_clst >= sectors) { + /* + * Convert mlcn/mlcn2 to sector positions before comparing with + * 'sectors'. All three are u64 values that come from the boot + * sector, so use check_mul_overflow() to keep a wraparound from + * silently bypassing the comparison. + */ + if (check_mul_overflow(mlcn, (u64)sct_per_clst, &mft_pos) || + check_mul_overflow(mlcn2, (u64)sct_per_clst, &mft2_pos) || + mft_pos >= sectors || mft2_pos >= sectors) { ntfs_err( sb, "%s: start of MFT 0x%llx (0x%llx) is out of volume 0x%llx.", diff --git a/fs/ntfs3/xattr.c b/fs/ntfs3/xattr.c index 04814dd2937529..6697362e655bf8 100644 --- a/fs/ntfs3/xattr.c +++ b/fs/ntfs3/xattr.c @@ -146,26 +146,29 @@ static int ntfs_read_ea(struct ntfs_inode *ni, struct EA_FULL **ea, for (off = 0; off < size; off += ea_size) { const struct EA_FULL *ef = Add2Ptr(ea_p, off); u32 bytes = size - off; + size_t need; /* Check if we can use field ea->size. */ if (bytes < sizeof(ef->size)) goto out1; + /* Check if we can use fields ef->name_len and ef->elength. */ + if (bytes < offsetof(struct EA_FULL, name)) + goto out1; + + /* Size needed to hold this record's name and value. */ + need = struct_size(ef, name, + 1 + ef->name_len + le16_to_cpu(ef->elength)); + if (ef->size) { ea_size = le32_to_cpu(ef->size); - if (ea_size > bytes) + /* ef->size must fit the list and cover the record. */ + if (ea_size > bytes || ea_size < need) goto out1; continue; } - /* Check if we can use fields ef->name_len and ef->elength. */ - if (bytes < offsetof(struct EA_FULL, name)) - goto out1; - - ea_size = ALIGN(struct_size(ef, name, - 1 + ef->name_len + - le16_to_cpu(ef->elength)), - 4); + ea_size = ALIGN(need, 4); if (ea_size > bytes) goto out1; } diff --git a/fs/ocfs2/acl.c b/fs/ocfs2/acl.c index af1e2cedb217f1..090ec60fb576a6 100644 --- a/fs/ocfs2/acl.c +++ b/fs/ocfs2/acl.c @@ -110,8 +110,7 @@ static void *ocfs2_acl_to_xattr(const struct posix_acl *acl, size_t *size) return ocfs2_acl; } -static struct posix_acl *ocfs2_get_acl_nolock(struct inode *inode, - int type, +static struct posix_acl *ocfs2_get_acl_nolock(struct inode *inode, int type, struct buffer_head *di_bh) { int name_index; @@ -349,63 +348,105 @@ int ocfs2_acl_chmod(struct inode *inode, struct buffer_head *bh) * Initialize the ACLs of a new inode. If parent directory has default ACL, * then clone to new inode. Called from ocfs2_mknod. */ -int ocfs2_init_acl(handle_t *handle, - struct inode *inode, - struct inode *dir, - struct buffer_head *di_bh, - struct buffer_head *dir_bh, - struct ocfs2_alloc_context *meta_ac, - struct ocfs2_alloc_context *data_ac) +void ocfs2_acl_init_release(struct ocfs2_acl_state *state) +{ + posix_acl_release(state->default_acl); + posix_acl_release(state->acl); + state->default_acl = NULL; + state->acl = NULL; +} + +int ocfs2_acl_init_prepare(struct inode *inode, struct inode *dir, + struct buffer_head *dir_bh, + struct ocfs2_acl_state *state) { struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); - struct posix_acl *acl = NULL; - int ret = 0, ret2; - umode_t mode; - - if (!S_ISLNK(inode->i_mode)) { - if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { - down_read(&OCFS2_I(dir)->ip_xattr_sem); - acl = ocfs2_get_acl_nolock(dir, ACL_TYPE_DEFAULT, - dir_bh); - up_read(&OCFS2_I(dir)->ip_xattr_sem); - if (IS_ERR(acl)) - return PTR_ERR(acl); + int ret = 0; + + state->default_acl = NULL; + state->acl = NULL; + state->mode = inode->i_mode; + + if (S_ISLNK(inode->i_mode)) + return 0; + + if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { + down_read(&OCFS2_I(dir)->ip_xattr_sem); + state->default_acl = + ocfs2_get_acl_nolock(dir, ACL_TYPE_DEFAULT, dir_bh); + up_read(&OCFS2_I(dir)->ip_xattr_sem); + if (IS_ERR(state->default_acl)) { + ret = PTR_ERR(state->default_acl); + state->default_acl = NULL; + return ret; } - if (!acl) { - mode = inode->i_mode & ~current_umask(); - ret = ocfs2_acl_set_mode(inode, di_bh, handle, mode); - if (ret) { - mlog_errno(ret); + if (state->default_acl) { + state->acl = posix_acl_dup(state->default_acl); + if (!state->acl) { + ret = -ENOMEM; goto cleanup; } + ret = __posix_acl_create(&state->acl, GFP_NOFS, + &state->mode); + if (ret < 0) + goto cleanup; + if (ret == 0) { + posix_acl_release(state->acl); + state->acl = NULL; + } + if (!S_ISDIR(inode->i_mode)) { + posix_acl_release(state->default_acl); + state->default_acl = NULL; + } + } else { + state->mode &= ~current_umask(); } + } else { + state->mode &= ~current_umask(); } - if ((osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) && acl) { - if (S_ISDIR(inode->i_mode)) { + + return 0; +cleanup: + ocfs2_acl_init_release(state); + return ret; +} + +int ocfs2_init_acl(handle_t *handle, struct inode *inode, + struct buffer_head *di_bh, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_acl_state *state) +{ + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + int ret = 0; + + if (S_ISLNK(inode->i_mode)) + return 0; + + if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { + if (S_ISDIR(inode->i_mode) && state->default_acl) { ret = ocfs2_set_acl(handle, inode, di_bh, - ACL_TYPE_DEFAULT, acl, - meta_ac, data_ac); + ACL_TYPE_DEFAULT, + state->default_acl, meta_ac, + data_ac); if (ret) - goto cleanup; + return ret; } - mode = inode->i_mode; - ret = __posix_acl_create(&acl, GFP_NOFS, &mode); - if (ret < 0) - return ret; + } - ret2 = ocfs2_acl_set_mode(inode, di_bh, handle, mode); - if (ret2) { - mlog_errno(ret2); - ret = ret2; - goto cleanup; - } - if (ret > 0) { - ret = ocfs2_set_acl(handle, inode, - di_bh, ACL_TYPE_ACCESS, - acl, meta_ac, data_ac); + ret = ocfs2_acl_set_mode(inode, di_bh, handle, state->mode); + if (ret) { + mlog_errno(ret); + return ret; + } + + if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { + if (state->acl) { + ret = ocfs2_set_acl(handle, inode, di_bh, + ACL_TYPE_ACCESS, state->acl, + meta_ac, data_ac); } } -cleanup: - posix_acl_release(acl); + return ret; } diff --git a/fs/ocfs2/acl.h b/fs/ocfs2/acl.h index 667c6f03fa606f..a91f9ce278d6d3 100644 --- a/fs/ocfs2/acl.h +++ b/fs/ocfs2/acl.h @@ -20,9 +20,20 @@ struct posix_acl *ocfs2_iop_get_acl(struct inode *inode, int type, bool rcu); int ocfs2_iop_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, struct posix_acl *acl, int type); extern int ocfs2_acl_chmod(struct inode *, struct buffer_head *); -extern int ocfs2_init_acl(handle_t *, struct inode *, struct inode *, - struct buffer_head *, struct buffer_head *, - struct ocfs2_alloc_context *, - struct ocfs2_alloc_context *); +struct ocfs2_acl_state { + struct posix_acl *default_acl; + struct posix_acl *acl; + umode_t mode; +}; + +int ocfs2_acl_init_prepare(struct inode *inode, struct inode *dir, + struct buffer_head *dir_bh, + struct ocfs2_acl_state *state); +void ocfs2_acl_init_release(struct ocfs2_acl_state *state); +int ocfs2_init_acl(handle_t *handle, struct inode *inode, + struct buffer_head *di_bh, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_acl_state *state); #endif /* OCFS2_ACL_H */ diff --git a/fs/ocfs2/cluster/heartbeat.c b/fs/ocfs2/cluster/heartbeat.c index ea68dc0887f623..8b5d8c2d7d3671 100644 --- a/fs/ocfs2/cluster/heartbeat.c +++ b/fs/ocfs2/cluster/heartbeat.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -211,6 +212,7 @@ struct o2hb_region { /* protected by the hr_callback_sem */ struct task_struct *hr_task; + u8 hr_node_num; unsigned int hr_blocks; unsigned long long hr_start_block; @@ -265,6 +267,9 @@ struct o2hb_region { /* Message key for negotiate timeout message. */ unsigned int hr_key; struct list_head hr_handler_list; + /* Serializes timeout arming against failed-start and teardown. */ + struct mutex hr_arming_mutex; + bool hr_stopping; /* last hb status, 0 for success, other value for error. */ int hr_last_hb_status; @@ -329,9 +334,14 @@ static void o2hb_write_timeout(struct work_struct *work) static void o2hb_arm_timeout(struct o2hb_region *reg) { + mutex_lock(®->hr_arming_mutex); + + if (reg->hr_stopping) + goto out_unlock; + /* Arm writeout only after thread reaches steady state */ if (atomic_read(®->hr_steady_iterations) != 0) - return; + goto out_unlock; mlog(ML_HEARTBEAT, "Queue write timeout for %u ms\n", O2HB_MAX_WRITE_TIMEOUT_MS); @@ -350,6 +360,18 @@ static void o2hb_arm_timeout(struct o2hb_region *reg) schedule_delayed_work(®->hr_nego_timeout_work, msecs_to_jiffies(O2HB_NEGO_TIMEOUT_MS)); bitmap_zero(reg->hr_nego_node_bitmap, O2NM_MAX_NODES); + +out_unlock: + mutex_unlock(®->hr_arming_mutex); +} + +static void o2hb_queue_nego_timeout(struct o2hb_region *reg, + unsigned long delay) +{ + mutex_lock(®->hr_arming_mutex); + if (!reg->hr_stopping) + schedule_delayed_work(®->hr_nego_timeout_work, delay); + mutex_unlock(®->hr_arming_mutex); } static void o2hb_disarm_timeout(struct o2hb_region *reg) @@ -358,12 +380,25 @@ static void o2hb_disarm_timeout(struct o2hb_region *reg) cancel_delayed_work_sync(®->hr_nego_timeout_work); } -static int o2hb_send_nego_msg(int key, int type, u8 target) +static void o2hb_set_region_stopping(struct o2hb_region *reg, bool stopping) +{ + mutex_lock(®->hr_arming_mutex); + reg->hr_stopping = stopping; + mutex_unlock(®->hr_arming_mutex); +} + +static void o2hb_quiesce_timeout(struct o2hb_region *reg) +{ + o2hb_set_region_stopping(reg, true); + o2hb_disarm_timeout(reg); +} + +static int o2hb_send_nego_msg(int key, int type, u8 target, u8 node_num) { struct o2hb_nego_msg msg; int status, ret; - msg.node_num = o2nm_this_node(); + msg.node_num = node_num; again: ret = o2net_send_message(type, key, &msg, sizeof(msg), target, &status); @@ -381,8 +416,10 @@ static void o2hb_nego_timeout(struct work_struct *work) unsigned long live_node_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; int master_node, i, ret; struct o2hb_region *reg; + u8 node_num; reg = container_of(work, struct o2hb_region, hr_nego_timeout_work.work); + node_num = reg->hr_node_num; /* don't negotiate timeout if last hb failed since it is very * possible io failed. Should let write timeout fence self. */ @@ -393,10 +430,10 @@ static void o2hb_nego_timeout(struct work_struct *work) /* lowest node as master node to make negotiate decision. */ master_node = find_first_bit(live_node_bitmap, O2NM_MAX_NODES); - if (master_node == o2nm_this_node()) { + if (master_node == node_num) { if (!test_bit(master_node, reg->hr_nego_node_bitmap)) { printk(KERN_NOTICE "o2hb: node %d hb write hung for %ds on region %s (%pg).\n", - o2nm_this_node(), O2HB_NEGO_TIMEOUT_MS/1000, + node_num, O2HB_NEGO_TIMEOUT_MS / 1000, config_item_name(®->hr_item), reg_bdev(reg)); set_bit(master_node, reg->hr_nego_node_bitmap); } @@ -405,8 +442,7 @@ static void o2hb_nego_timeout(struct work_struct *work) /* check negotiate bitmap every second to do timeout * approve decision. */ - schedule_delayed_work(®->hr_nego_timeout_work, - msecs_to_jiffies(1000)); + o2hb_queue_nego_timeout(reg, msecs_to_jiffies(1000)); return; } @@ -425,7 +461,7 @@ static void o2hb_nego_timeout(struct work_struct *work) mlog(ML_HEARTBEAT, "send NEGO_APPROVE msg to node %d\n", i); ret = o2hb_send_nego_msg(reg->hr_key, - O2HB_NEGO_APPROVE_MSG, i); + O2HB_NEGO_APPROVE_MSG, i, node_num); if (ret) mlog(ML_ERROR, "send NEGO_APPROVE msg to node %d fail %d\n", i, ret); @@ -433,10 +469,10 @@ static void o2hb_nego_timeout(struct work_struct *work) } else { /* negotiate timeout with master node. */ printk(KERN_NOTICE "o2hb: node %d hb write hung for %ds on region %s (%pg), negotiate timeout with node %d.\n", - o2nm_this_node(), O2HB_NEGO_TIMEOUT_MS/1000, config_item_name(®->hr_item), + node_num, O2HB_NEGO_TIMEOUT_MS / 1000, config_item_name(®->hr_item), reg_bdev(reg), master_node); ret = o2hb_send_nego_msg(reg->hr_key, O2HB_NEGO_TIMEOUT_MSG, - master_node); + master_node, node_num); if (ret) mlog(ML_ERROR, "send NEGO_TIMEOUT msg to node %d fail %d\n", master_node, ret); @@ -609,7 +645,9 @@ static int o2hb_issue_node_write(struct o2hb_region *reg, o2hb_bio_wait_init(write_wc); - slot = o2nm_this_node(); + slot = reg->hr_node_num; + if (slot >= O2NM_MAX_NODES) + return -EINVAL; bio = o2hb_setup_one_bio(reg, write_wc, &slot, slot+1, REQ_OP_WRITE | REQ_SYNC); @@ -678,8 +716,12 @@ static int o2hb_check_own_slot(struct o2hb_region *reg) struct o2hb_disk_slot *slot; struct o2hb_disk_heartbeat_block *hb_block; char *errstr; + u8 node_num = reg->hr_node_num; - slot = ®->hr_slots[o2nm_this_node()]; + if (node_num >= O2NM_MAX_NODES) + return 0; + + slot = ®->hr_slots[node_num]; /* Don't check on our 1st timestamp */ if (!slot->ds_last_time) return 0; @@ -720,7 +762,10 @@ static inline void o2hb_prepare_block(struct o2hb_region *reg, struct o2hb_disk_slot *slot; struct o2hb_disk_heartbeat_block *hb_block; - node_num = o2nm_this_node(); + node_num = reg->hr_node_num; + if (node_num >= O2NM_MAX_NODES) + return; + slot = ®->hr_slots[node_num]; hb_block = (struct o2hb_disk_heartbeat_block *)slot->ds_raw_block; @@ -1214,7 +1259,7 @@ static int o2hb_thread(void *data) set_user_nice(current, MIN_NICE); /* Pin node */ - ret = o2nm_depend_this_node(); + ret = o2nm_depend_node(reg->hr_node_num); if (ret) { mlog(ML_ERROR, "Node has been deleted, ret = %d\n", ret); reg->hr_node_deleted = 1; @@ -1223,7 +1268,8 @@ static int o2hb_thread(void *data) } while (!kthread_should_stop() && - !reg->hr_unclean_stop && !reg->hr_aborted_start) { + !reg->hr_unclean_stop && !reg->hr_aborted_start && + o2nm_this_node() == reg->hr_node_num) { /* We track the time spent inside * o2hb_do_disk_heartbeat so that we avoid more than * hr_timeout_ms between disk writes. On busy systems @@ -1272,7 +1318,7 @@ static int o2hb_thread(void *data) } /* Unpin node */ - o2nm_undepend_this_node(); + o2nm_undepend_node(reg->hr_node_num); mlog(ML_HEARTBEAT|ML_KTHREAD, "o2hb thread exiting\n"); @@ -1464,13 +1510,38 @@ void o2hb_init(void) o2hb_debug_init(); } -/* if we're already in a callback then we're already serialized by the sem */ -static void o2hb_fill_node_map_from_callback(unsigned long *map, - unsigned int bits) +static void __o2hb_fill_node_map(unsigned long *map, unsigned int bits) { bitmap_copy(map, o2hb_live_node_bitmap, bits); } +void o2hb_callback_read_lock(void) +{ + down_read(&o2hb_callback_sem); +} + +void o2hb_callback_read_unlock(void) +{ + up_read(&o2hb_callback_sem); +} + +void o2hb_synchronize_callbacks(void) +{ + down_write(&o2hb_callback_sem); + up_write(&o2hb_callback_sem); +} + +/* + * Callers must already hold o2hb_callback_sem for read or write so the copy + * stays serialized with callback delivery. + */ +void o2hb_fill_node_map_locked(unsigned long *map, unsigned int bits) +{ + spin_lock(&o2hb_live_lock); + __o2hb_fill_node_map(map, bits); + spin_unlock(&o2hb_live_lock); +} + /* * get a map of all nodes that are heartbeating in any regions */ @@ -1478,11 +1549,9 @@ void o2hb_fill_node_map(unsigned long *map, unsigned int bits) { /* callers want to serialize this map and callbacks so that they * can trust that they don't miss nodes coming to the party */ - down_read(&o2hb_callback_sem); - spin_lock(&o2hb_live_lock); - o2hb_fill_node_map_from_callback(map, bits); - spin_unlock(&o2hb_live_lock); - up_read(&o2hb_callback_sem); + o2hb_callback_read_lock(); + o2hb_fill_node_map_locked(map, bits); + o2hb_callback_read_unlock(); } EXPORT_SYMBOL_GPL(o2hb_fill_node_map); @@ -1530,6 +1599,8 @@ static void o2hb_region_release(struct config_item *item) mlog(ML_HEARTBEAT, "hb region release (%pg)\n", reg_bdev(reg)); + o2hb_quiesce_timeout(reg); + o2net_unregister_and_flush_handler_list(®->hr_handler_list); o2hb_unmap_slot_data(reg); if (reg->hr_bdev_file) @@ -1545,7 +1616,6 @@ static void o2hb_region_release(struct config_item *item) list_del(®->hr_all_item); spin_unlock(&o2hb_live_lock); - o2net_unregister_handler_list(®->hr_handler_list); kfree(reg); } @@ -1799,7 +1869,8 @@ static ssize_t o2hb_region_dev_store(struct config_item *item, /* We can't heartbeat without having had our node number * configured yet. */ - if (o2nm_this_node() == O2NM_MAX_NODES) + reg->hr_node_num = o2nm_this_node(); + if (reg->hr_node_num == O2NM_MAX_NODES) return -EINVAL; ret = kstrtol(p, 0, &fd); @@ -1859,9 +1930,6 @@ static ssize_t o2hb_region_dev_store(struct config_item *item, goto out; } - INIT_DELAYED_WORK(®->hr_write_timeout_work, o2hb_write_timeout); - INIT_DELAYED_WORK(®->hr_nego_timeout_work, o2hb_nego_timeout); - /* * A node is considered live after it has beat LIVE_THRESHOLD * times. We're not steady until we've given them a chance @@ -1881,6 +1949,7 @@ static ssize_t o2hb_region_dev_store(struct config_item *item, atomic_set(®->hr_steady_iterations, live_threshold); /* unsteady_iterations is triple the steady_iterations */ atomic_set(®->hr_unsteady_iterations, (live_threshold * 3)); + o2hb_set_region_stopping(reg, false); hb_task = kthread_run(o2hb_thread, reg, "o2hb-%s", reg->hr_item.ci_name); @@ -1930,6 +1999,8 @@ static ssize_t o2hb_region_dev_store(struct config_item *item, out: if (ret < 0) { + o2hb_quiesce_timeout(reg); + spin_lock(&o2hb_live_lock); hb_task = reg->hr_task; reg->hr_task = NULL; @@ -2044,6 +2115,7 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *g ret = -ENAMETOOLONG; goto free; } + reg->hr_node_num = O2NM_MAX_NODES; spin_lock(&o2hb_live_lock); reg->hr_region_num = 0; @@ -2068,6 +2140,10 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *g */ reg->hr_key = crc32_le(reg->hr_region_num + O2NM_MAX_REGIONS, name, strlen(name)); + mutex_init(®->hr_arming_mutex); + reg->hr_stopping = true; + INIT_DELAYED_WORK(®->hr_write_timeout_work, o2hb_write_timeout); + INIT_DELAYED_WORK(®->hr_nego_timeout_work, o2hb_nego_timeout); INIT_LIST_HEAD(®->hr_handler_list); ret = o2net_register_handler(O2HB_NEGO_TIMEOUT_MSG, reg->hr_key, sizeof(struct o2hb_nego_msg), @@ -2088,7 +2164,7 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *g return ®->hr_item; unregister_handler: - o2net_unregister_handler_list(®->hr_handler_list); + o2net_unregister_and_flush_handler_list(®->hr_handler_list); remove_item: spin_lock(&o2hb_live_lock); list_del(®->hr_all_item); @@ -2107,6 +2183,8 @@ static void o2hb_heartbeat_group_drop_item(struct config_group *group, struct o2hb_region *reg = to_o2hb_region(item); int quorum_region = 0; + o2hb_quiesce_timeout(reg); + /* stop the thread when the user removes the region dir */ spin_lock(&o2hb_live_lock); hb_task = reg->hr_task; @@ -2579,7 +2657,7 @@ int o2hb_check_node_heartbeating_no_sem(u8 node_num) unsigned long testing_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; spin_lock(&o2hb_live_lock); - o2hb_fill_node_map_from_callback(testing_map, O2NM_MAX_NODES); + __o2hb_fill_node_map(testing_map, O2NM_MAX_NODES); spin_unlock(&o2hb_live_lock); if (!test_bit(node_num, testing_map)) { mlog(ML_HEARTBEAT, @@ -2596,7 +2674,7 @@ int o2hb_check_node_heartbeating_from_callback(u8 node_num) { unsigned long testing_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; - o2hb_fill_node_map_from_callback(testing_map, O2NM_MAX_NODES); + o2hb_fill_node_map_locked(testing_map, O2NM_MAX_NODES); if (!test_bit(node_num, testing_map)) { mlog(ML_HEARTBEAT, "node (%u) does not have heartbeating enabled.\n", diff --git a/fs/ocfs2/cluster/heartbeat.h b/fs/ocfs2/cluster/heartbeat.h index 8ef8c1b9eeb767..2ca2b657583c03 100644 --- a/fs/ocfs2/cluster/heartbeat.h +++ b/fs/ocfs2/cluster/heartbeat.h @@ -58,6 +58,11 @@ int o2hb_register_callback(const char *region_uuid, struct o2hb_callback_func *hc); void o2hb_unregister_callback(const char *region_uuid, struct o2hb_callback_func *hc); +void o2hb_callback_read_lock(void); +void o2hb_callback_read_unlock(void); +void o2hb_synchronize_callbacks(void); +void o2hb_fill_node_map_locked(unsigned long *map, + unsigned int bits); void o2hb_fill_node_map(unsigned long *map, unsigned int bits); void o2hb_exit(void); diff --git a/fs/ocfs2/cluster/nodemanager.c b/fs/ocfs2/cluster/nodemanager.c index 21154e4baddf4d..e08850a5d7364c 100644 --- a/fs/ocfs2/cluster/nodemanager.c +++ b/fs/ocfs2/cluster/nodemanager.c @@ -326,6 +326,7 @@ static ssize_t o2nm_node_local_store(struct config_item *item, const char *page, struct o2nm_node *node = to_o2nm_node(item); struct o2nm_cluster *cluster; unsigned long tmp; + bool starting = false; char *p = (char *)page; ssize_t ret; @@ -362,11 +363,13 @@ static ssize_t o2nm_node_local_store(struct config_item *item, const char *page, ret = o2net_start_listening(node); if (ret) goto out; + starting = true; } if (!tmp && cluster->cl_has_local && cluster->cl_local_node == node->nd_num) { o2net_stop_listening(node); + cluster->cl_has_local = 0; cluster->cl_local_node = O2NM_INVALID_NODE_NUM; } @@ -374,6 +377,8 @@ static ssize_t o2nm_node_local_store(struct config_item *item, const char *page, if (node->nd_local) { cluster->cl_has_local = tmp; cluster->cl_local_node = node->nd_num; + if (starting) + o2net_complete_start_listening(node); } ret = count; @@ -788,12 +793,12 @@ void o2nm_undepend_item(struct config_item *item) configfs_undepend_item(item); } -int o2nm_depend_this_node(void) +int o2nm_depend_node(u8 node_num) { int ret = 0; struct o2nm_node *local_node; - local_node = o2nm_get_node_by_num(o2nm_this_node()); + local_node = o2nm_get_node_by_num(node_num); if (!local_node) { ret = -EINVAL; goto out; @@ -806,17 +811,27 @@ out: return ret; } -void o2nm_undepend_this_node(void) +void o2nm_undepend_node(u8 node_num) { struct o2nm_node *local_node; - local_node = o2nm_get_node_by_num(o2nm_this_node()); + local_node = o2nm_get_node_by_num(node_num); BUG_ON(!local_node); o2nm_undepend_item(&local_node->nd_item); o2nm_node_put(local_node); } +int o2nm_depend_this_node(void) +{ + return o2nm_depend_node(o2nm_this_node()); +} + +void o2nm_undepend_this_node(void) +{ + o2nm_undepend_node(o2nm_this_node()); +} + static void __exit exit_o2nm(void) { diff --git a/fs/ocfs2/cluster/nodemanager.h b/fs/ocfs2/cluster/nodemanager.h index 2f72f56996bd9e..ca3483fb545044 100644 --- a/fs/ocfs2/cluster/nodemanager.h +++ b/fs/ocfs2/cluster/nodemanager.h @@ -66,6 +66,8 @@ void o2nm_node_put(struct o2nm_node *node); int o2nm_depend_item(struct config_item *item); int o2nm_depend_item_unlocked(struct config_item *item); void o2nm_undepend_item(struct config_item *item); +int o2nm_depend_node(u8 node_num); +void o2nm_undepend_node(u8 node_num); int o2nm_depend_this_node(void); void o2nm_undepend_this_node(void); diff --git a/fs/ocfs2/cluster/tcp.c b/fs/ocfs2/cluster/tcp.c index 09a1f3b77bb8b8..474fe1414cee86 100644 --- a/fs/ocfs2/cluster/tcp.c +++ b/fs/ocfs2/cluster/tcp.c @@ -38,6 +38,8 @@ */ #include +#include +#include #include #include #include @@ -102,9 +104,16 @@ static struct socket *o2net_listen_sock; * quorum work is queued as sock containers are shutdown.. stop_listening * tears down all the node's sock containers, preventing future shutdowns * and queued quorum work, before canceling delayed quorum work and - * destroying the work queue. + * destroying the work queue. Handler teardown can also race local listener + * shutdown, so keep a waitable destroying pointer until the old ordered + * queue has finished draining. */ static struct workqueue_struct *o2net_wq; +static struct workqueue_struct *o2net_wq_destroying; +static DEFINE_MUTEX(o2net_wq_mutex); +static DECLARE_COMPLETION(o2net_wq_destroyed); +/* Heartbeat callbacks stay registered across local-node off/on. */ +static bool o2net_listening; static struct work_struct o2net_listen_work; static struct o2hb_callback_func o2net_hb_up, o2net_hb_down; @@ -884,6 +893,27 @@ void o2net_unregister_handler_list(struct list_head *list) } EXPORT_SYMBOL_GPL(o2net_unregister_handler_list); +static void o2net_flush_wq(void) +{ + mutex_lock(&o2net_wq_mutex); + if (o2net_wq_destroying) { + mutex_unlock(&o2net_wq_mutex); + wait_for_completion(&o2net_wq_destroyed); + return; + } + + if (o2net_wq) + flush_workqueue(o2net_wq); + mutex_unlock(&o2net_wq_mutex); +} + +void o2net_unregister_and_flush_handler_list(struct list_head *list) +{ + o2net_unregister_handler_list(list); + o2net_flush_wq(); +} +EXPORT_SYMBOL_GPL(o2net_unregister_and_flush_handler_list); + static struct o2net_msg_handler *o2net_handler_get(u32 msg_type, u32 key) { struct o2net_msg_handler *nmh; @@ -1692,6 +1722,19 @@ static void o2net_still_up(struct work_struct *work) /* ------------------------------------------------------------ */ +static void o2net_hb_node_up(struct o2net_node *nn) +{ + /* ensure an immediate connect attempt */ + nn->nn_last_connect_attempt = jiffies - + (msecs_to_jiffies(o2net_reconnect_delay()) + 1); + + spin_lock(&nn->nn_lock); + atomic_set(&nn->nn_timeout, 0); + if (nn->nn_persistent_error) + o2net_set_nn_state(nn, NULL, 0, 0); + spin_unlock(&nn->nn_lock); +} + void o2net_disconnect_node(struct o2nm_node *node) { struct o2net_node *nn = o2net_nn_from_num(node->nd_num); @@ -1702,52 +1745,48 @@ void o2net_disconnect_node(struct o2nm_node *node) o2net_set_nn_state(nn, NULL, 0, -ENOTCONN); spin_unlock(&nn->nn_lock); - if (o2net_wq) { - cancel_delayed_work(&nn->nn_connect_expired); - cancel_delayed_work(&nn->nn_connect_work); - cancel_delayed_work(&nn->nn_still_up); - flush_workqueue(o2net_wq); - } + cancel_delayed_work(&nn->nn_connect_expired); + cancel_delayed_work(&nn->nn_connect_work); + cancel_delayed_work(&nn->nn_still_up); + o2net_flush_wq(); } static void o2net_hb_node_down_cb(struct o2nm_node *node, int node_num, void *data) { + u8 this_node; + o2quo_hb_down(node_num); if (!node) - return; + goto out; + + this_node = o2nm_this_node(); + if (!READ_ONCE(o2net_listening) || this_node == O2NM_MAX_NODES) + goto out; - if (node_num != o2nm_this_node()) + if (node_num != this_node) o2net_disconnect_node(node); +out: BUG_ON(atomic_read(&o2net_connected_peers) < 0); } static void o2net_hb_node_up_cb(struct o2nm_node *node, int node_num, void *data) { - struct o2net_node *nn = o2net_nn_from_num(node_num); + u8 this_node; o2quo_hb_up(node_num); BUG_ON(!node); - /* ensure an immediate connect attempt */ - nn->nn_last_connect_attempt = jiffies - - (msecs_to_jiffies(o2net_reconnect_delay()) + 1); + this_node = o2nm_this_node(); + if (!READ_ONCE(o2net_listening) || this_node == O2NM_MAX_NODES) + return; - if (node_num != o2nm_this_node()) { - /* believe it or not, accept and node heartbeating testing - * can succeed for this node before we got here.. so - * only use set_nn_state to clear the persistent error - * if that hasn't already happened */ - spin_lock(&nn->nn_lock); - atomic_set(&nn->nn_timeout, 0); - if (nn->nn_persistent_error) - o2net_set_nn_state(nn, NULL, 0, 0); - spin_unlock(&nn->nn_lock); - } + if (node_num != this_node) + o2net_hb_node_up(o2net_nn_from_num(node_num)); } void o2net_unregister_hb_callbacks(void) @@ -1756,6 +1795,37 @@ void o2net_unregister_hb_callbacks(void) o2hb_unregister_callback(NULL, &o2net_hb_down); } +/* + * Delay heartbeat-driven network work until the local node is fully published + * through o2nm_this_node(), then replay the nodes that are already live while + * callback delivery stays blocked. + */ +void o2net_complete_start_listening(struct o2nm_node *node) +{ + unsigned long live_nodes[BITS_TO_LONGS(O2NM_MAX_NODES)]; + unsigned long node_num; + u8 local_node; + + local_node = o2nm_this_node(); + if (WARN_ON_ONCE(local_node == O2NM_MAX_NODES)) + return; + if (WARN_ON_ONCE(local_node != node->nd_num)) + return; + if (WARN_ON_ONCE(!o2net_wq)) + return; + + o2hb_callback_read_lock(); + WRITE_ONCE(o2net_listening, true); + o2hb_fill_node_map_locked(live_nodes, O2NM_MAX_NODES); + for_each_set_bit(node_num, live_nodes, O2NM_MAX_NODES) { + if (node_num == local_node) + continue; + + o2net_hb_node_up(o2net_nn_from_num(node_num)); + } + o2hb_callback_read_unlock(); +} + int o2net_register_hb_callbacks(void) { int ret; @@ -2023,6 +2093,36 @@ out: return ret; } +static void o2net_destroy_wq(void) +{ + struct workqueue_struct *wq; + + mutex_lock(&o2net_wq_mutex); + if (o2net_wq_destroying) { + mutex_unlock(&o2net_wq_mutex); + wait_for_completion(&o2net_wq_destroyed); + return; + } + + wq = o2net_wq; + if (!wq) { + mutex_unlock(&o2net_wq_mutex); + return; + } + + reinit_completion(&o2net_wq_destroyed); + o2net_wq_destroying = wq; + mutex_unlock(&o2net_wq_mutex); + + destroy_workqueue(wq); + + mutex_lock(&o2net_wq_mutex); + o2net_wq = NULL; + o2net_wq_destroying = NULL; + complete_all(&o2net_wq_destroyed); + mutex_unlock(&o2net_wq_mutex); +} + /* * called from node manager when we should bring up our network listening * socket. node manager handles all the serialization to only call this @@ -2033,22 +2133,44 @@ out: int o2net_start_listening(struct o2nm_node *node) { int ret = 0; + struct workqueue_struct *wq; + + if (WARN_ON_ONCE(READ_ONCE(o2net_listening))) + return -EBUSY; + + mutex_lock(&o2net_wq_mutex); + if (o2net_wq_destroying) { + mutex_unlock(&o2net_wq_mutex); + return -EBUSY; + } + if (WARN_ON_ONCE(o2net_wq)) { + mutex_unlock(&o2net_wq_mutex); + return -EBUSY; + } + mutex_unlock(&o2net_wq_mutex); - BUG_ON(o2net_wq != NULL); BUG_ON(o2net_listen_sock != NULL); mlog(ML_KTHREAD, "starting o2net thread...\n"); - o2net_wq = alloc_ordered_workqueue("o2net", WQ_MEM_RECLAIM); - if (o2net_wq == NULL) { + wq = alloc_ordered_workqueue("o2net", WQ_MEM_RECLAIM); + if (!wq) { mlog(ML_ERROR, "unable to launch o2net thread\n"); return -ENOMEM; /* ? */ } + mutex_lock(&o2net_wq_mutex); + if (unlikely(o2net_wq_destroying || o2net_wq)) { + mutex_unlock(&o2net_wq_mutex); + destroy_workqueue(wq); + return -EBUSY; + } + o2net_wq = wq; + mutex_unlock(&o2net_wq_mutex); + ret = o2net_open_listening_sock(node->nd_ipv4_address, node->nd_ipv4_port); if (ret) { - destroy_workqueue(o2net_wq); - o2net_wq = NULL; + o2net_destroy_wq(); } else o2quo_conn_up(node->nd_num); @@ -2065,6 +2187,9 @@ void o2net_stop_listening(struct o2nm_node *node) BUG_ON(o2net_wq == NULL); BUG_ON(o2net_listen_sock == NULL); + WRITE_ONCE(o2net_listening, false); + o2hb_synchronize_callbacks(); + /* stop the listening socket from generating work */ write_lock_bh(&sock->sk->sk_callback_lock); sock->sk->sk_data_ready = sock->sk->sk_user_data; @@ -2081,8 +2206,7 @@ void o2net_stop_listening(struct o2nm_node *node) /* finish all work and tear down the work queue */ mlog(ML_KTHREAD, "waiting for o2net thread to exit....\n"); - destroy_workqueue(o2net_wq); - o2net_wq = NULL; + o2net_destroy_wq(); sock_release(o2net_listen_sock); o2net_listen_sock = NULL; diff --git a/fs/ocfs2/cluster/tcp.h b/fs/ocfs2/cluster/tcp.h index a75b551d31c7be..a11bcee289479f 100644 --- a/fs/ocfs2/cluster/tcp.h +++ b/fs/ocfs2/cluster/tcp.h @@ -89,6 +89,7 @@ int o2net_register_handler(u32 msg_type, u32 key, u32 max_len, o2net_post_msg_handler_func *post_func, struct list_head *unreg_list); void o2net_unregister_handler_list(struct list_head *list); +void o2net_unregister_and_flush_handler_list(struct list_head *list); void o2net_fill_node_map(unsigned long *map, unsigned bytes); @@ -96,6 +97,7 @@ struct o2nm_node; int o2net_register_hb_callbacks(void); void o2net_unregister_hb_callbacks(void); int o2net_start_listening(struct o2nm_node *node); +void o2net_complete_start_listening(struct o2nm_node *node); void o2net_stop_listening(struct o2nm_node *node); void o2net_disconnect_node(struct o2nm_node *node); int o2net_num_connected_peers(void); diff --git a/fs/ocfs2/inode.c b/fs/ocfs2/inode.c index 662dbc845b8bd2..180107a11046c5 100644 --- a/fs/ocfs2/inode.c +++ b/fs/ocfs2/inode.c @@ -1528,6 +1528,22 @@ int ocfs2_validate_inode_block(struct super_block *sb, goto bail; } + if ((le32_to_cpu(di->i_flags) & OCFS2_ORPHANED_FL) && + le16_to_cpu(di->i_orphaned_slot) >= OCFS2_SB(sb)->max_slots) { + rc = ocfs2_error(sb, "Invalid dinode %llu: orphaned slot %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_orphaned_slot)); + goto bail; + } + + if ((le32_to_cpu(di->i_flags) & OCFS2_DIO_ORPHANED_FL) && + le16_to_cpu(di->i_dio_orphaned_slot) >= OCFS2_SB(sb)->max_slots) { + rc = ocfs2_error(sb, "Invalid dinode %llu: DIO orphaned slot %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_dio_orphaned_slot)); + goto bail; + } + /* * Reject dinodes whose i_mode does not name one of the seven * canonical POSIX file types. ocfs2_populate_inode() copies @@ -1608,6 +1624,10 @@ int ocfs2_validate_inode_block(struct super_block *sb, goto bail; } + rc = ocfs2_validate_inode_xattr(sb, bh->b_blocknr, di); + if (rc) + goto bail; + if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) { struct ocfs2_inline_data *data = &di->id2.i_data; @@ -1948,8 +1968,6 @@ int ocfs2_read_inode_block_full(struct inode *inode, struct buffer_head **bh, rc = ocfs2_read_blocks(INODE_CACHE(inode), OCFS2_I(inode)->ip_blkno, 1, &tmp, flags, ocfs2_validate_inode_block); - if (rc < 0) - make_bad_inode(inode); /* If ocfs2_read_blocks() got us a new bh, pass it up. */ if (!rc && !*bh) *bh = tmp; diff --git a/fs/ocfs2/namei.c b/fs/ocfs2/namei.c index 1277666c77cd2d..ea37a50580890c 100644 --- a/fs/ocfs2/namei.c +++ b/fs/ocfs2/namei.c @@ -256,6 +256,7 @@ static int ocfs2_mknod(struct mnt_idmap *idmap, sigset_t oldset; int did_block_signals = 0; struct ocfs2_dentry_lock *dl = NULL; + struct ocfs2_acl_state acl_state = { 0 }; trace_ocfs2_mknod(dir, dentry, dentry->d_name.len, dentry->d_name.name, (unsigned long long)OCFS2_I(dir)->ip_blkno, @@ -330,10 +331,14 @@ static int ocfs2_mknod(struct mnt_idmap *idmap, } } + status = ocfs2_acl_init_prepare(inode, dir, parent_fe_bh, &acl_state); + if (status < 0) + goto leave; + /* calculate meta data/clusters for setting security and acl xattr */ - status = ocfs2_calc_xattr_init(dir, parent_fe_bh, mode, - &si, &want_clusters, - &xattr_credits, &want_meta); + status = ocfs2_calc_xattr_init(dir, mode, &si, &want_clusters, + &xattr_credits, &want_meta, + &acl_state); if (status < 0) { mlog_errno(status); goto leave; @@ -411,8 +416,8 @@ static int ocfs2_mknod(struct mnt_idmap *idmap, inc_nlink(dir); } - status = ocfs2_init_acl(handle, inode, dir, new_fe_bh, parent_fe_bh, - meta_ac, data_ac); + status = ocfs2_init_acl(handle, inode, new_fe_bh, meta_ac, data_ac, + &acl_state); if (status < 0) { mlog_errno(status); @@ -477,6 +482,8 @@ leave: brelse(parent_fe_bh); kfree(si.value); + ocfs2_acl_init_release(&acl_state); + ocfs2_free_dir_lookup_result(&lookup); if (inode_ac) diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c index f1b3aaab6a7140..35bcbb0ff607b2 100644 --- a/fs/ocfs2/xattr.c +++ b/fs/ocfs2/xattr.c @@ -390,6 +390,12 @@ static int ocfs2_init_xattr_bucket(struct ocfs2_xattr_bucket *bucket, return rc; } +static int ocfs2_validate_xattr_entries_flat(struct super_block *sb, u64 blkno, + struct ocfs2_xattr_header *xh, + size_t region_size); +static int ocfs2_validate_xattr_bucket(struct ocfs2_xattr_bucket *bucket, + u64 blkno); + /* Read the xattr bucket at xb_blkno */ static int ocfs2_read_xattr_bucket(struct ocfs2_xattr_bucket *bucket, u64 xb_blkno) @@ -408,6 +414,8 @@ static int ocfs2_read_xattr_bucket(struct ocfs2_xattr_bucket *bucket, spin_unlock(&OCFS2_SB(bucket->bu_inode->i_sb)->osb_xattr_lock); if (rc) mlog_errno(rc); + else + rc = ocfs2_validate_xattr_bucket(bucket, xb_blkno); } if (rc) @@ -509,6 +517,22 @@ static int ocfs2_validate_xattr_block(struct super_block *sb, le32_to_cpu(xb->xb_fs_generation)); } + if (!(le16_to_cpu(xb->xb_flags) & OCFS2_XATTR_INDEXED)) { + size_t region_offset = + offsetof(struct ocfs2_xattr_block, xb_attrs.xb_header); + + if (bh->b_size < region_offset) + return ocfs2_error(sb, + "Invalid xattr block %llu: block size %zu is too small\n", + (unsigned long long)bh->b_blocknr, + bh->b_size); + + return ocfs2_validate_xattr_entries_flat(sb, bh->b_blocknr, + &xb->xb_attrs.xb_header, + bh->b_size - + region_offset); + } + return 0; } @@ -611,13 +635,10 @@ int ocfs2_calc_security_init(struct inode *dir, return ret; } -int ocfs2_calc_xattr_init(struct inode *dir, - struct buffer_head *dir_bh, - umode_t mode, +int ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, struct ocfs2_security_xattr_info *si, - int *want_clusters, - int *xattr_credits, - int *want_meta) + int *want_clusters, int *xattr_credits, + int *want_meta, struct ocfs2_acl_state *acl_state) { int ret = 0; struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); @@ -628,19 +649,15 @@ int ocfs2_calc_xattr_init(struct inode *dir, si->value_len); if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { - down_read(&OCFS2_I(dir)->ip_xattr_sem); - acl_len = ocfs2_xattr_get_nolock(dir, dir_bh, - OCFS2_XATTR_INDEX_POSIX_ACL_DEFAULT, - "", NULL, 0); - up_read(&OCFS2_I(dir)->ip_xattr_sem); - if (acl_len > 0) { - a_size = ocfs2_xattr_entry_real_size(0, acl_len); - if (S_ISDIR(mode)) - a_size <<= 1; - } else if (acl_len != 0 && acl_len != -ENODATA) { - ret = acl_len; - mlog_errno(ret); - return ret; + if (acl_state->default_acl && S_ISDIR(mode)) { + acl_len = acl_state->default_acl->a_count * + sizeof(struct ocfs2_acl_entry); + a_size += ocfs2_xattr_entry_real_size(0, acl_len); + } + if (acl_state->acl) { + acl_len = acl_state->acl->a_count * + sizeof(struct ocfs2_acl_entry); + a_size += ocfs2_xattr_entry_real_size(0, acl_len); } } @@ -683,14 +700,33 @@ int ocfs2_calc_xattr_init(struct inode *dir, new_clusters); *want_clusters += new_clusters; } - if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL && - acl_len > OCFS2_XATTR_INLINE_SIZE) { - /* for directory, it has DEFAULT and ACCESS two types of acls */ - new_clusters = (S_ISDIR(mode) ? 2 : 1) * - ocfs2_clusters_for_bytes(dir->i_sb, acl_len); - *xattr_credits += ocfs2_clusters_to_blocks(dir->i_sb, - new_clusters); - *want_clusters += new_clusters; + if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { + if (acl_state->default_acl && S_ISDIR(mode)) { + acl_len = acl_state->default_acl->a_count * + sizeof(struct ocfs2_acl_entry); + if (acl_len > OCFS2_XATTR_INLINE_SIZE) { + new_clusters = + ocfs2_clusters_for_bytes(dir->i_sb, + acl_len); + *xattr_credits += + ocfs2_clusters_to_blocks(dir->i_sb, + new_clusters); + *want_clusters += new_clusters; + } + } + if (acl_state->acl) { + acl_len = acl_state->acl->a_count * + sizeof(struct ocfs2_acl_entry); + if (acl_len > OCFS2_XATTR_INLINE_SIZE) { + new_clusters = + ocfs2_clusters_for_bytes(dir->i_sb, + acl_len); + *xattr_credits += + ocfs2_clusters_to_blocks(dir->i_sb, + new_clusters); + *want_clusters += new_clusters; + } + } } return ret; @@ -948,41 +984,206 @@ static int ocfs2_xattr_list_entries(struct inode *inode, return result; } -static int ocfs2_xattr_ibody_lookup_header(struct inode *inode, - struct ocfs2_dinode *di, - struct ocfs2_xattr_header **header) +static int ocfs2_validate_xattr_entries_flat(struct super_block *sb, u64 blkno, + struct ocfs2_xattr_header *xh, + size_t region_size) +{ + u16 xattr_count = le16_to_cpu(xh->xh_count); + size_t entries_limit = region_size; + size_t nv_limit = region_size; + size_t max_entries; + int i; + + if (region_size < sizeof(*xh)) + return ocfs2_error(sb, + "Invalid xattr in block %llu: region size %zu is too small\n", + (unsigned long long)blkno, region_size); + + max_entries = (entries_limit - sizeof(*xh)) / + sizeof(struct ocfs2_xattr_entry); + + if (xattr_count > max_entries) + return ocfs2_error(sb, + "Invalid xattr in block %llu: entry count %u exceeds maximum %zu\n", + (unsigned long long)blkno, + xattr_count, max_entries); + + for (i = 0; i < xattr_count; i++) { + struct ocfs2_xattr_entry *xe = &xh->xh_entries[i]; + size_t name_offset = le16_to_cpu(xe->xe_name_offset); + size_t value_offset; + + if (name_offset > nv_limit || + xe->xe_name_len > nv_limit - name_offset) + return ocfs2_error(sb, + "Invalid xattr in block %llu: entry %d name is out of bounds\n", + (unsigned long long)blkno, i); + + value_offset = name_offset + OCFS2_XATTR_SIZE(xe->xe_name_len); + if (value_offset > nv_limit) + return ocfs2_error(sb, + "Invalid xattr in block %llu: entry %d value starts out of bounds\n", + (unsigned long long)blkno, i); + + if (ocfs2_xattr_is_local(xe)) { + if (le64_to_cpu(xe->xe_value_size) > + nv_limit - value_offset) + return ocfs2_error(sb, + "Invalid xattr in block %llu: entry %d value is out of bounds\n", + (unsigned long long)blkno, + i); + } else if (sizeof(struct ocfs2_xattr_value_root) > + nv_limit - value_offset) { + return ocfs2_error(sb, + "Invalid xattr in block %llu: entry %d value root is out of bounds\n", + (unsigned long long)blkno, i); + } + } + + return 0; +} + +static int ocfs2_xattr_ibody_lookup_header_raw(struct super_block *sb, + u64 blkno, + struct ocfs2_dinode *di, + struct ocfs2_xattr_header **header, + u16 *inline_size_ret) { + struct ocfs2_xattr_header *xh; u16 xattr_count; size_t max_entries; u16 inline_size = le16_to_cpu(di->i_xattr_inline_size); - if (inline_size > inode->i_sb->s_blocksize || + if (inline_size > sb->s_blocksize || inline_size < sizeof(struct ocfs2_xattr_header)) { - ocfs2_error(inode->i_sb, - "Invalid xattr inline size %u in inode %llu\n", - inline_size, - (unsigned long long)OCFS2_I(inode)->ip_blkno); + ocfs2_error(sb, + "Invalid inode %llu: xattr inline size %u\n", + (unsigned long long)blkno, inline_size); return -EFSCORRUPTED; } - *header = (struct ocfs2_xattr_header *) - ((void *)di + inode->i_sb->s_blocksize - inline_size); + xh = (struct ocfs2_xattr_header *) + ((void *)di + sb->s_blocksize - inline_size); - xattr_count = le16_to_cpu((*header)->xh_count); + xattr_count = le16_to_cpu(xh->xh_count); max_entries = (inline_size - sizeof(struct ocfs2_xattr_header)) / sizeof(struct ocfs2_xattr_entry); if (xattr_count > max_entries) { - ocfs2_error(inode->i_sb, + ocfs2_error(sb, "xattr entry count %u exceeds maximum %zu in inode %llu\n", xattr_count, max_entries, - (unsigned long long)OCFS2_I(inode)->ip_blkno); + (unsigned long long)blkno); return -EFSCORRUPTED; } + *header = xh; + if (inline_size_ret) + *inline_size_ret = inline_size; + return 0; } +int ocfs2_validate_inode_xattr(struct super_block *sb, u64 blkno, + struct ocfs2_dinode *di) +{ + struct ocfs2_xattr_header *xh; + u16 inline_size; + int ret; + + if (!(le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_XATTR_FL)) + return 0; + + ret = ocfs2_xattr_ibody_lookup_header_raw(sb, blkno, di, &xh, + &inline_size); + if (ret) + return ret; + + return ocfs2_validate_xattr_entries_flat(sb, blkno, xh, inline_size); +} + +static int ocfs2_validate_xattr_bucket(struct ocfs2_xattr_bucket *bucket, + u64 blkno) +{ + struct super_block *sb = bucket->bu_inode->i_sb; + struct ocfs2_xattr_header *xh = bucket_xh(bucket); + u16 xattr_count = le16_to_cpu(xh->xh_count); + size_t region_size = (size_t)sb->s_blocksize * bucket->bu_blocks; + size_t entries_limit = sb->s_blocksize; + size_t nv_limit = sb->s_blocksize; + size_t max_entries; + int i; + + if (region_size < sizeof(*xh)) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: region size %zu is too small\n", + (unsigned long long)blkno, region_size); + + if (entries_limit < sizeof(*xh)) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entries limit %zu is too small\n", + (unsigned long long)blkno, + entries_limit); + + max_entries = (entries_limit - sizeof(*xh)) / + sizeof(struct ocfs2_xattr_entry); + + if (xattr_count > max_entries) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entry count %u exceeds maximum %zu\n", + (unsigned long long)blkno, + xattr_count, max_entries); + + for (i = 0; i < xattr_count; i++) { + struct ocfs2_xattr_entry *xe = &xh->xh_entries[i]; + size_t name_offset = le16_to_cpu(xe->xe_name_offset); + size_t block_off = name_offset >> sb->s_blocksize_bits; + size_t block_offset = name_offset % nv_limit; + size_t value_offset; + + if (name_offset >= region_size || block_off >= bucket->bu_blocks) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entry %d name is out of bounds\n", + (unsigned long long)blkno, i); + + if (xe->xe_name_len > nv_limit - block_offset) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entry %d name crosses block boundary\n", + (unsigned long long)blkno, i); + + value_offset = block_offset + OCFS2_XATTR_SIZE(xe->xe_name_len); + if (value_offset > nv_limit) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entry %d value starts out of bounds\n", + (unsigned long long)blkno, i); + + if (ocfs2_xattr_is_local(xe)) { + if (le64_to_cpu(xe->xe_value_size) > + nv_limit - value_offset) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entry %d value is out of bounds\n", + (unsigned long long)blkno, + i); + } else if (sizeof(struct ocfs2_xattr_value_root) > + nv_limit - value_offset) { + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entry %d value root is out of bounds\n", + (unsigned long long)blkno, i); + } + } + + return 0; +} + +static int ocfs2_xattr_ibody_lookup_header(struct inode *inode, + struct ocfs2_dinode *di, + struct ocfs2_xattr_header **header) +{ + return ocfs2_xattr_ibody_lookup_header_raw(inode->i_sb, + OCFS2_I(inode)->ip_blkno, + di, header, NULL); +} + int ocfs2_has_inline_xattr_value_outside(struct inode *inode, struct ocfs2_dinode *di) { @@ -3489,9 +3690,10 @@ out: } /* - * This function only called duing creating inode - * for init security/acl xattrs of the new inode. - * All transanction credits have been reserved in mknod. + * This helper is only for setting initial ACL or security xattrs on an inode + * that is still unpublished, unhashed, and unattached to a dentry. + * Ordinary xattr updates must use ocfs2_xattr_set(). + * All transaction credits have been reserved in mknod or symlink callers. */ int ocfs2_xattr_set_handle(handle_t *handle, struct inode *inode, @@ -3548,8 +3750,6 @@ int ocfs2_xattr_set_handle(handle_t *handle, xis.inode_bh = xbs.inode_bh = di_bh; di = (struct ocfs2_dinode *)di_bh->b_data; - down_write(&OCFS2_I(inode)->ip_xattr_sem); - ret = ocfs2_xattr_ibody_find(inode, name_index, name, &xis); if (ret) goto cleanup; @@ -3562,7 +3762,6 @@ int ocfs2_xattr_set_handle(handle_t *handle, ret = __ocfs2_xattr_set_handle(inode, di, &xi, &xis, &xbs, &ctxt); cleanup: - up_write(&OCFS2_I(inode)->ip_xattr_sem); brelse(xbs.xattr_bh); ocfs2_xattr_bucket_free(xbs.bucket); @@ -7262,6 +7461,7 @@ int ocfs2_init_security_and_acl(struct inode *dir, { int ret = 0; struct buffer_head *dir_bh = NULL; + struct ocfs2_acl_state acl_state = { 0 }; ret = ocfs2_init_security_get(inode, dir, qstr, NULL); if (ret) { @@ -7274,10 +7474,17 @@ int ocfs2_init_security_and_acl(struct inode *dir, mlog_errno(ret); goto leave; } - ret = ocfs2_init_acl(NULL, inode, dir, NULL, dir_bh, NULL, NULL); + + ret = ocfs2_acl_init_prepare(inode, dir, dir_bh, &acl_state); + if (ret) + goto unlock; + + ret = ocfs2_init_acl(NULL, inode, NULL, NULL, NULL, &acl_state); if (ret) mlog_errno(ret); +unlock: + ocfs2_acl_init_release(&acl_state); ocfs2_inode_unlock(dir, 0); brelse(dir_bh); leave: diff --git a/fs/ocfs2/xattr.h b/fs/ocfs2/xattr.h index 65e9aa7439199e..5e18513277f18b 100644 --- a/fs/ocfs2/xattr.h +++ b/fs/ocfs2/xattr.h @@ -43,6 +43,8 @@ int ocfs2_xattr_set_handle(handle_t *, struct inode *, struct buffer_head *, struct ocfs2_alloc_context *); int ocfs2_has_inline_xattr_value_outside(struct inode *inode, struct ocfs2_dinode *di); +int ocfs2_validate_inode_xattr(struct super_block *sb, u64 blkno, + struct ocfs2_dinode *di); int ocfs2_xattr_remove(struct inode *, struct buffer_head *); int ocfs2_init_security_get(struct inode *, struct inode *, const struct qstr *, @@ -55,9 +57,12 @@ int ocfs2_init_security_set(handle_t *, struct inode *, int ocfs2_calc_security_init(struct inode *, struct ocfs2_security_xattr_info *, int *, int *, struct ocfs2_alloc_context **); -int ocfs2_calc_xattr_init(struct inode *, struct buffer_head *, - umode_t, struct ocfs2_security_xattr_info *, - int *, int *, int *); + +struct ocfs2_acl_state; +int ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, + struct ocfs2_security_xattr_info *si, + int *want_clusters, int *xattr_credits, + int *want_meta, struct ocfs2_acl_state *acl_state); /* * xattrs can live inside an inode, as part of an external xattr block, diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c index a932a1fea81828..723ba366324a8a 100644 --- a/fs/resctrl/monitor.c +++ b/fs/resctrl/monitor.c @@ -628,14 +628,22 @@ void mon_event_count(void *info) rr->err = 0; } -static struct rdt_ctrl_domain *get_ctrl_domain_from_cpu(int cpu, - struct rdt_resource *r) +/* + * Find the software controller's ctrl domain that contains @cpu on resource @r. + * + * Only called from the mbm_over worker via update_mba_bw() where the returned + * domain is kept alive by cancel_delayed_work_sync() in + * resctrl_offline_ctrl_domain(). This drains this worker and then waits on + * rdtgroup_mutex held here before the architecture can free the ctrl domain. + * + * Context: Call from RCU read-side critical section. + */ +static struct rdt_ctrl_domain *get_sc_ctrl_domain_from_cpu(int cpu, + struct rdt_resource *r) { struct rdt_ctrl_domain *d; - lockdep_assert_cpus_held(); - - list_for_each_entry(d, &r->ctrl_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list) { /* Find the domain that contains this CPU */ if (cpumask_test_cpu(cpu, &d->hdr.cpu_mask)) return d; @@ -696,7 +704,8 @@ static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_l3_mon_domain *dom_m if (WARN_ON_ONCE(!pmbm_data)) return; - dom_mba = get_ctrl_domain_from_cpu(smp_processor_id(), r_mba); + guard(rcu)(); + dom_mba = get_sc_ctrl_domain_from_cpu(smp_processor_id(), r_mba); if (!dom_mba) { pr_warn_once("Failure to get domain for MBA update\n"); return; @@ -799,11 +808,25 @@ void cqm_handle_limbo(struct work_struct *work) unsigned long delay = msecs_to_jiffies(CQM_LIMBOCHECK_INTERVAL); struct rdt_l3_mon_domain *d; - cpus_read_lock(); + /* + * Safe to run without CPU hotplug lock. Work is guaranteed to be + * canceled before the domain structure is removed. + */ mutex_lock(&rdtgroup_mutex); + /* + * Ensure the worker is dedicated to a CPU as intended and not + * relocated by workqueue subsystem as part of CPU going offline. + */ + if (!is_percpu_thread()) + goto out_unlock; + d = container_of(work, struct rdt_l3_mon_domain, cqm_limbo.work); + /* Domain is going offline */ + if (cpumask_empty(&d->hdr.cpu_mask)) + goto out_unlock; + __check_limbo(d, false); if (has_busy_rmid(d)) { @@ -813,8 +836,8 @@ void cqm_handle_limbo(struct work_struct *work) delay); } +out_unlock: mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); } /** @@ -846,7 +869,10 @@ void mbm_handle_overflow(struct work_struct *work) struct list_head *head; struct rdt_resource *r; - cpus_read_lock(); + /* + * Safe to run without CPU hotplug lock. Work is guaranteed to be + * canceled before the domain structure is removed. + */ mutex_lock(&rdtgroup_mutex); /* @@ -856,9 +882,24 @@ void mbm_handle_overflow(struct work_struct *work) if (!resctrl_mounted || !resctrl_arch_mon_capable()) goto out_unlock; + /* + * Ensure the worker is dedicated to a CPU and not relocated by + * workqueue subsystem as part of CPU going offline since reading + * events depend on smp_processor_id(). After passing this check + * smp_processor_id() is valid for entire duration of this worker + * since it runs with rdtgroup_mutex held and the offline handler needs + * rdtgroup_mutex to offline the CPU being run on here. + */ + if (!is_percpu_thread()) + goto out_unlock; + r = resctrl_arch_get_resource(RDT_RESOURCE_L3); d = container_of(work, struct rdt_l3_mon_domain, mbm_over.work); + /* Domain is going offline */ + if (cpumask_empty(&d->hdr.cpu_mask)) + goto out_unlock; + list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) { mbm_update(r, d, prgrp); @@ -880,7 +921,6 @@ void mbm_handle_overflow(struct work_struct *work) out_unlock: mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); } /** diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index cc9966ff6cdfb6..11c88d593f1efe 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -2608,15 +2608,24 @@ static void rdtgroup_kn_get(struct rdtgroup *rdtgrp, struct kernfs_node *kn) static void rdtgroup_kn_put(struct rdtgroup *rdtgrp, struct kernfs_node *kn) { - if (atomic_dec_and_test(&rdtgrp->waitcount) && - (rdtgrp->flags & RDT_DELETED)) { + bool needs_free; + + if (!atomic_dec_and_mutex_lock(&rdtgrp->waitcount, &rdtgroup_mutex)) { + kernfs_unbreak_active_protection(kn); + return; + } + + needs_free = rdtgrp->flags & RDT_DELETED; + + mutex_unlock(&rdtgroup_mutex); + + kernfs_unbreak_active_protection(kn); + + if (needs_free) { if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) rdtgroup_pseudo_lock_remove(rdtgrp); - kernfs_unbreak_active_protection(kn); rdtgroup_remove(rdtgrp); - } else { - kernfs_unbreak_active_protection(kn); } } @@ -4333,6 +4342,29 @@ static void domain_destroy_l3_mon_state(struct rdt_l3_mon_domain *d) void resctrl_offline_ctrl_domain(struct rdt_resource *r, struct rdt_ctrl_domain *d) { + /* + * mbm_handle_overflow() may dereference this ctrl domain via + * update_mba_bw()->get_sc_ctrl_domain_from_cpu(). The architecture has + * unlinked the domain from the RCU list and waited a grace period, so + * no new worker iteration can find it; drain any worker that already + * holds a pointer to it before the architecture frees the domain. + * + * Software controller is enabled/disabled on mount/unmount with + * cpus_read_lock() held. Running here with cpus_write_lock() so + * there are no concurrent changes to software controller status. + */ + if (r->rid == RDT_RESOURCE_MBA && is_mba_sc(r)) { + struct rdt_resource *l3 = resctrl_arch_get_resource(RDT_RESOURCE_L3); + struct rdt_l3_mon_domain *mon_d; + + list_for_each_entry_rcu(mon_d, &l3->mon_domains, hdr.list, lockdep_is_cpus_held()) { + if (mon_d->hdr.id == d->hdr.id) { + cancel_delayed_work_sync(&mon_d->mbm_over); + break; + } + } + } + mutex_lock(&rdtgroup_mutex); if (supports_mba_mbps() && r->rid == RDT_RESOURCE_MBA) @@ -4345,6 +4377,24 @@ void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_domain_hdr *h { struct rdt_l3_mon_domain *d; + /* + * Called by architecture under CPU hotplug lock as it prepares to remove + * the domain which is guaranteed to be accessible here. + * The domain has been unlinked from the RCU list and a grace period + * has elapsed, so no new worker can be scheduled. Drain any worker that + * is in flight or pending before letting architecture proceed to free + * the domain that has the workers' struct delayed_work embedded. + * Do so before taking rdtgroup_mutex since the workers also acquire it. + */ + if (r->rid == RDT_RESOURCE_L3 && + domain_header_is_valid(hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_L3)) { + d = container_of(hdr, struct rdt_l3_mon_domain, hdr); + if (resctrl_is_mbm_enabled()) + cancel_delayed_work_sync(&d->mbm_over); + if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID)) + cancel_delayed_work_sync(&d->cqm_limbo); + } + mutex_lock(&rdtgroup_mutex); /* @@ -4361,8 +4411,6 @@ void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_domain_hdr *h goto out_unlock; d = container_of(hdr, struct rdt_l3_mon_domain, hdr); - if (resctrl_is_mbm_enabled()) - cancel_delayed_work(&d->mbm_over); if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID) && has_busy_rmid(d)) { /* * When a package is going down, forcefully @@ -4373,7 +4421,6 @@ void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_domain_hdr *h * package never comes back. */ __check_limbo(d, true); - cancel_delayed_work(&d->cqm_limbo); } domain_destroy_l3_mon_state(d); @@ -4554,12 +4601,16 @@ void resctrl_offline_cpu(unsigned int cpu) d = get_mon_domain_from_cpu(cpu, l3); if (d) { if (resctrl_is_mbm_enabled() && cpu == d->mbm_work_cpu) { - cancel_delayed_work(&d->mbm_over); + mutex_unlock(&rdtgroup_mutex); + cancel_delayed_work_sync(&d->mbm_over); + mutex_lock(&rdtgroup_mutex); mbm_setup_overflow_handler(d, 0, cpu); } if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID) && cpu == d->cqm_work_cpu && has_busy_rmid(d)) { - cancel_delayed_work(&d->cqm_limbo); + mutex_unlock(&rdtgroup_mutex); + cancel_delayed_work_sync(&d->cqm_limbo); + mutex_lock(&rdtgroup_mutex); cqm_setup_limbo_handler(d, 0, cpu); } } diff --git a/fs/smb/client/cifs_debug.c b/fs/smb/client/cifs_debug.c index 4ed4f55a0bb72a..3761d3ad60882f 100644 --- a/fs/smb/client/cifs_debug.c +++ b/fs/smb/client/cifs_debug.c @@ -754,7 +754,7 @@ static ssize_t cifs_stats_proc_write(struct file *file, atomic_set(&server->smb2slowcmd[i], 0); server->time_per_cmd[i] = 0; server->slowest_cmd[i] = 0; - server->fastest_cmd[0] = 0; + server->fastest_cmd[i] = 0; } #endif /* CONFIG_CIFS_STATS2 */ list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { diff --git a/fs/smb/client/dir.c b/fs/smb/client/dir.c index 88a4a1787ff047..b0ddcaa2d81593 100644 --- a/fs/smb/client/dir.c +++ b/fs/smb/client/dir.c @@ -1138,6 +1138,8 @@ int cifs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, } while (unlikely(rc == -EEXIST) && ++retries < max_retries); if (rc) { + if (rc == -ENOENT) + rc = -EOPNOTSUPP; cifs_del_pending_open(&open); goto out; } diff --git a/fs/smb/client/inode.c b/fs/smb/client/inode.c index aec8ed8881a6c3..85ef8fff0e8422 100644 --- a/fs/smb/client/inode.c +++ b/fs/smb/client/inode.c @@ -3310,13 +3310,6 @@ cifs_setattr_unix(struct dentry *direntry, struct iattr *attrs) if (rc) goto out; - if ((attrs->ia_valid & ATTR_SIZE) && - attrs->ia_size != i_size_read(inode)) { - truncate_setsize(inode, attrs->ia_size); - netfs_resize_file(&cifsInode->netfs, attrs->ia_size, true); - fscache_resize_cookie(cifs_inode_cookie(inode), attrs->ia_size); - } - setattr_copy(&nop_mnt_idmap, inode, attrs); mark_inode_dirty(inode); @@ -3537,13 +3530,6 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) if (rc) goto cifs_setattr_exit; - if ((attrs->ia_valid & ATTR_SIZE) && - attrs->ia_size != i_size_read(inode)) { - truncate_setsize(inode, attrs->ia_size); - netfs_resize_file(&cifsInode->netfs, attrs->ia_size, true); - fscache_resize_cookie(cifs_inode_cookie(inode), attrs->ia_size); - } - setattr_copy(&nop_mnt_idmap, inode, attrs); mark_inode_dirty(inode); diff --git a/fs/smb/client/smb2inode.c b/fs/smb/client/smb2inode.c index 2946391bb992d0..457442af1222ef 100644 --- a/fs/smb/client/smb2inode.c +++ b/fs/smb/client/smb2inode.c @@ -576,6 +576,7 @@ finished: idata->fi.Attributes = create_rsp->FileAttributes; idata->fi.AllocationSize = create_rsp->AllocationSize; idata->fi.EndOfFile = create_rsp->EndofFile; + idata->contains_posix_file_info = false; if (le32_to_cpu(idata->fi.NumberOfLinks) == 0) idata->fi.NumberOfLinks = cpu_to_le32(1); /* dummy value */ idata->fi.DeletePending = 0; /* successful open = not delete pending */ @@ -598,7 +599,6 @@ finished: switch (cmds[i]) { case SMB2_OP_QUERY_INFO: idata = in_iov[i].iov_base; - idata->contains_posix_file_info = false; if (rc == 0 && cfile && cfile->symlink_target) { idata->symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL); if (!idata->symlink_target) @@ -611,6 +611,8 @@ finished: le16_to_cpu(qi_rsp->OutputBufferOffset), le32_to_cpu(qi_rsp->OutputBufferLength), &rsp_iov[i + 1], sizeof(idata->fi), (char *)&idata->fi); + if (!rc) + idata->contains_posix_file_info = false; } SMB2_query_info_free(&rqst[num_rqst++]); if (rc) @@ -622,7 +624,6 @@ finished: break; case SMB2_OP_POSIX_QUERY_INFO: idata = in_iov[i].iov_base; - idata->contains_posix_file_info = true; if (rc == 0 && cfile && cfile->symlink_target) { idata->symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL); if (!idata->symlink_target) @@ -636,6 +637,8 @@ finished: le32_to_cpu(qi_rsp->OutputBufferLength), &rsp_iov[i + 1], sizeof(idata->posix_fi) /* add SIDs */, (char *)&idata->posix_fi); + if (!rc) + idata->contains_posix_file_info = true; } if (rc == 0) rc = parse_posix_sids(idata, &rsp_iov[i + 1]); @@ -707,7 +710,6 @@ finished: idata = in_iov[i].iov_base; idata->reparse.io.iov = *iov; idata->reparse.io.buftype = resp_buftype[i + 1]; - idata->contains_posix_file_info = false; /* BB VERIFY */ rbuf = reparse_buf_ptr(iov); if (IS_ERR(rbuf)) { rc = PTR_ERR(rbuf); @@ -729,7 +731,6 @@ finished: case SMB2_OP_QUERY_WSL_EA: if (!rc) { idata = in_iov[i].iov_base; - idata->contains_posix_file_info = false; qi_rsp = rsp_iov[i + 1].iov_base; data[0] = (u8 *)qi_rsp + le16_to_cpu(qi_rsp->OutputBufferOffset); size[0] = le32_to_cpu(qi_rsp->OutputBufferLength); diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c index 4ce165e40657f3..dea05aeb53a18d 100644 --- a/fs/smb/client/smb2pdu.c +++ b/fs/smb/client/smb2pdu.c @@ -3372,6 +3372,7 @@ replay_again: #endif /* CIFS_DEBUG2 */ if (file_info) { + buf->contains_posix_file_info = false; file_info->CreationTime = rsp->CreationTime; file_info->LastAccessTime = rsp->LastAccessTime; file_info->LastWriteTime = rsp->LastWriteTime; @@ -4564,8 +4565,10 @@ smb2_new_read_req(void **buf, unsigned int *total_len, if (rc) return rc; - if (server == NULL) - return -ECONNABORTED; + if (!server) { + rc = -ECONNABORTED; + goto free_req; + } shdr = &req->hdr; shdr->Id.SyncId.ProcessId = cpu_to_le32(io_parms->pid); @@ -4596,8 +4599,10 @@ smb2_new_read_req(void **buf, unsigned int *total_len, rdata->mr = smbd_register_mr(server->smbd_conn, &rdata->subreq.io_iter, true, need_invalidate); - if (!rdata->mr) - return -EAGAIN; + if (!rdata->mr) { + rc = -EAGAIN; + goto free_req; + } req->Channel = SMB2_CHANNEL_RDMA_V1_INVALIDATE; if (need_invalidate) @@ -4638,6 +4643,10 @@ smb2_new_read_req(void **buf, unsigned int *total_len, *buf = req; return rc; + +free_req: + cifs_small_buf_release(req); + return rc; } static void @@ -4885,6 +4894,7 @@ out: smb2_should_replay(tcon, &rdata->retries, &rdata->cur_sleep)) { + rdata->replay = true; trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_retry_needed); __set_bit(NETFS_SREQ_NEED_RETRY, &rdata->subreq.flags); } diff --git a/fs/smb/common/smb2pdu.h b/fs/smb/common/smb2pdu.h index e7ff52b8aba5f9..37b660b2047b09 100644 --- a/fs/smb/common/smb2pdu.h +++ b/fs/smb/common/smb2pdu.h @@ -847,8 +847,8 @@ struct smb2_lock_req { __le16 StructureSize; /* Must be 48 */ __le16 LockCount; /* - * The least significant four bits are the index, the other 28 bits are - * the lock sequence number (0 to 64). See MS-SMB2 2.2.26 + * The least significant four bits are the lock sequence number. The + * other 28 bits are the index (0 to 64). See MS-SMB2 2.2.26. */ __le32 LockSequenceNumber; __u64 PersistentFileId; diff --git a/fs/smb/common/smbglob.h b/fs/smb/common/smbglob.h index 4e33d91cdc9dbe..d9c7e6e7af295c 100644 --- a/fs/smb/common/smbglob.h +++ b/fs/smb/common/smbglob.h @@ -39,6 +39,7 @@ struct smb_version_values { size_t create_mxac_size; size_t create_disk_id_size; size_t create_posix_size; + size_t create_aapl_size; }; static inline unsigned int get_rfc1002_len(void *buf) diff --git a/fs/smb/server/Kconfig b/fs/smb/server/Kconfig index 08d8b7a965a64d..221ec9717a8341 100644 --- a/fs/smb/server/Kconfig +++ b/fs/smb/server/Kconfig @@ -9,7 +9,6 @@ config SMB_SERVER select CRYPTO select CRYPTO_LIB_AES_CBC_MACS select CRYPTO_LIB_ARC4 - select CRYPTO_LIB_DES select CRYPTO_LIB_MD5 select CRYPTO_LIB_SHA256 select CRYPTO_LIB_SHA512 diff --git a/fs/smb/server/auth.c b/fs/smb/server/auth.c index 4e7b6f0e6b8cd8..8d22d28a12bf9c 100644 --- a/fs/smb/server/auth.c +++ b/fs/smb/server/auth.c @@ -24,7 +24,6 @@ #include #include -#include #include "server.h" #include "smb_common.h" @@ -439,6 +438,7 @@ int ksmbd_krb5_authenticate(struct ksmbd_session *sess, char *in_blob, resp_ext = ksmbd_ipc_login_request_ext(resp->login_response.account); user = ksmbd_alloc_user(&resp->login_response, resp_ext); + kvfree(resp_ext); if (!user) { ksmbd_debug(AUTH, "login failure\n"); retval = -ENOMEM; @@ -462,6 +462,7 @@ int ksmbd_krb5_authenticate(struct ksmbd_session *sess, char *in_blob, memcpy(out_blob, resp->payload + resp->session_key_len, resp->spnego_blob_len); *out_len = resp->spnego_blob_len; + sess->kerberos_expiry = resp->session_expiry; retval = 0; out: kvfree(resp); @@ -716,8 +717,21 @@ static int ksmbd_get_encryption_key(struct ksmbd_work *work, __u64 ses_id, if (enc) sess = work->sess; - else - sess = ksmbd_session_lookup_all(work->conn, ses_id); + else { + /* + * A previous-session replacement leaves the old encryption key in + * place. Use it to authenticate an encrypted request, then let + * session validation reject the expired session. This preserves the + * encrypted STATUS_USER_SESSION_DELETED response without reviving + * the session. + */ + sess = ksmbd_session_lookup_all_states(work->conn, ses_id); + if (sess && sess->state != SMB2_SESSION_VALID && + (sess->state != SMB2_SESSION_EXPIRED || !sess->enc)) { + ksmbd_user_session_put(sess); + sess = NULL; + } + } if (!sess) return -EINVAL; diff --git a/fs/smb/server/compress.c b/fs/smb/server/compress.c index 01d1771ff6636b..5162fb84c755ab 100644 --- a/fs/smb/server/compress.c +++ b/fs/smb/server/compress.c @@ -14,24 +14,14 @@ #define SMB_COMPRESS_MIN_LEN PAGE_SIZE -/** - * ksmbd_decompress_request() - replace a compressed request with its SMB2 PDU - * @conn: connection which owns the current RFC1002 request buffer - * - * Derive the uncompressed size from the transform variant, enforce ksmbd's - * normal message limits, and ask the common decoder to validate every payload. - * On success, replace conn->request_buf with a regular RFC1002-framed SMB2 - * message so the rest of the request path needs no compression awareness. - * - * Return: 0 on success, otherwise a negative errno. - */ -int ksmbd_decompress_request(struct ksmbd_conn *conn) +static int __ksmbd_decompress_request(struct ksmbd_conn *conn, + void *request_buf, void **out_buf) { struct smb2_compression_hdr *hdr; - unsigned int pdu_size = get_rfc1002_len(conn->request_buf); + unsigned int pdu_size = get_rfc1002_len(request_buf); u32 orig_size, offset, out_size; u32 max_allowed_pdu_size; - char *buf, *out; + char *out; int rc; if (pdu_size < sizeof(struct smb2_compression_hdr)) @@ -41,7 +31,7 @@ int ksmbd_decompress_request(struct ksmbd_conn *conn) conn->compress_algorithm == SMB3_COMPRESS_NONE) return -EINVAL; - hdr = smb_get_msg(conn->request_buf); + hdr = smb_get_msg(request_buf); if (hdr->ProtocolId != SMB2_COMPRESSION_TRANSFORM_ID) return -EINVAL; @@ -74,19 +64,69 @@ int ksmbd_decompress_request(struct ksmbd_conn *conn) if (!out) return -ENOMEM; - buf = (char *)hdr; *(__be32 *)out = cpu_to_be32(out_size); rc = smb_compression_decompress(conn->compress_algorithm, conn->compress_chained, conn->compress_pattern, - buf, pdu_size, out + 4, out_size); + (char *)hdr, pdu_size, out + 4, out_size); if (rc) { kvfree(out); return rc; } + *out_buf = out; + return 0; +} + +/** + * ksmbd_decompress_request() - replace a compressed request with its SMB2 PDU + * @conn: connection which owns the current RFC1002 request buffer + * + * Derive the uncompressed size from the transform variant, enforce ksmbd's + * normal message limits, and ask the common decoder to validate every payload. + * On success, replace conn->request_buf with a regular RFC1002-framed SMB2 + * message so the rest of the request path needs no compression awareness. + * + * Return: 0 on success, otherwise a negative errno. + */ +int ksmbd_decompress_request(struct ksmbd_conn *conn) +{ + void *out_buf; + int rc; + + rc = __ksmbd_decompress_request(conn, conn->request_buf, &out_buf); + if (rc) + return rc; + kvfree(conn->request_buf); - conn->request_buf = out; + conn->request_buf = out_buf; + return 0; +} + +/** + * ksmbd_decompress_work_request() - decompress an encrypted work request + * @work: work item whose request buffer contains a compression transform + * + * SMB3 encrypts a compressed message by applying compression first and + * encryption second. The receive loop can therefore only decode the + * compression transform before work allocation for an unencrypted request; + * an encrypted request must be decompressed after its encryption layer has + * been removed. + * + * Return: 0 on success, otherwise a negative errno. + */ +int ksmbd_decompress_work_request(struct ksmbd_work *work) +{ + void *out_buf; + int rc; + + rc = __ksmbd_decompress_request(work->conn, work->request_buf, + &out_buf); + if (rc) + return rc; + + kvfree(work->request_buf); + work->request_buf = out_buf; return 0; } diff --git a/fs/smb/server/compress.h b/fs/smb/server/compress.h index 663c6f44f09b01..13df2eb221e8c6 100644 --- a/fs/smb/server/compress.h +++ b/fs/smb/server/compress.h @@ -11,6 +11,7 @@ #include "../common/compress/compress.h" int ksmbd_decompress_request(struct ksmbd_conn *conn); +int ksmbd_decompress_work_request(struct ksmbd_work *work); int ksmbd_compress_response(struct ksmbd_work *work); #endif /* __KSMBD_COMPRESS_H__ */ diff --git a/fs/smb/server/connection.c b/fs/smb/server/connection.c index ef6f202f4024c0..af73c2ed5d249f 100644 --- a/fs/smb/server/connection.c +++ b/fs/smb/server/connection.c @@ -11,6 +11,7 @@ #include "server.h" #include "smb_common.h" #include "mgmt/ksmbd_ida.h" +#include "mgmt/user_session.h" #include "connection.h" #include "compress.h" #include "transport_tcp.h" @@ -185,6 +186,7 @@ void ksmbd_conn_free(struct ksmbd_conn *conn) kvfree(conn->request_buf); kfree(conn->preauth_info); kfree(conn->mechToken); + ksmbd_preauth_session_destroy(conn); ksmbd_conn_put(conn); } @@ -223,6 +225,7 @@ struct ksmbd_conn *ksmbd_conn_alloc(void) init_waitqueue_head(&conn->r_count_q); INIT_LIST_HEAD(&conn->requests); INIT_LIST_HEAD(&conn->async_requests); + INIT_LIST_HEAD(&conn->preauth_sess_table); spin_lock_init(&conn->request_lock); spin_lock_init(&conn->credits_lock); ida_init(&conn->async_ida); @@ -301,25 +304,64 @@ void ksmbd_conn_unlock(struct ksmbd_conn *conn) mutex_unlock(&conn->srv_mutex); } -void ksmbd_all_conn_set_status(u64 sess_id, u32 status) +static bool ksmbd_session_is_bound_to_conn(struct ksmbd_session *sess, + struct ksmbd_conn *conn) +{ + bool found; + + rcu_read_lock(); + found = xa_load(&conn->sessions, sess->id) == sess; + rcu_read_unlock(); + if (found) + return true; + + down_read(&sess->chann_lock); + found = xa_load(&sess->ksmbd_chann_list, (long)conn); + up_read(&sess->chann_lock); + return found; +} + +void ksmbd_all_conn_set_status(struct ksmbd_session *sess, u32 status) { struct ksmbd_conn *conn; int bkt; down_read(&conn_list_lock); hash_for_each(conn_list, bkt, conn, hlist) { - if (conn->binding || xa_load(&conn->sessions, sess_id)) - WRITE_ONCE(conn->status, status); + if (ksmbd_session_is_bound_to_conn(sess, conn)) { + spin_lock(&conn->request_lock); + if (!ksmbd_conn_exiting(conn) && + !ksmbd_conn_releasing(conn)) + WRITE_ONCE(conn->status, status); + spin_unlock(&conn->request_lock); + } } up_read(&conn_list_lock); } +void ksmbd_conn_abort(struct ksmbd_conn *conn) +{ + bool shutdown = false; + + spin_lock(&conn->request_lock); + if (!ksmbd_conn_exiting(conn) && !ksmbd_conn_releasing(conn)) { + ksmbd_conn_set_exiting(conn); + shutdown = true; + } + spin_unlock(&conn->request_lock); + wake_up_all(&conn->req_running_q); + + if (shutdown && conn->transport->ops->shutdown) + conn->transport->ops->shutdown(conn->transport); +} + void ksmbd_conn_wait_idle(struct ksmbd_conn *conn) { wait_event(conn->req_running_q, atomic_read(&conn->req_running) < 2); } -int ksmbd_conn_wait_idle_sess_id(struct ksmbd_conn *curr_conn, u64 sess_id) +int ksmbd_conn_wait_idle_sess(struct ksmbd_conn *curr_conn, + struct ksmbd_session *sess) { struct ksmbd_conn *conn; int rc, retry_count = 0, max_timeout = 120; @@ -331,7 +373,7 @@ retry_idle: down_read(&conn_list_lock); hash_for_each(conn_list, bkt, conn, hlist) { - if (conn->binding || xa_load(&conn->sessions, sess_id)) { + if (ksmbd_session_is_bound_to_conn(sess, conn)) { rcount = (conn == curr_conn) ? 2 : 1; if (atomic_read(&conn->req_running) >= rcount) { rc = wait_event_timeout(conn->req_running_q, @@ -623,7 +665,8 @@ int ksmbd_conn_transport_init(void) } out: mutex_unlock(&init_lock); - create_proc_clients(); + if (create_proc_clients()) + pr_warn("Unable to create clients procfs entry\n"); return ret; } @@ -657,8 +700,10 @@ again: * handler exited its receive loop for an unrelated * reason). */ - if (READ_ONCE(conn->status) != KSMBD_SESS_RELEASING) + spin_lock(&conn->request_lock); + if (!ksmbd_conn_releasing(conn)) ksmbd_conn_set_exiting(conn); + spin_unlock(&conn->request_lock); target = conn; break; } diff --git a/fs/smb/server/connection.h b/fs/smb/server/connection.h index 0e4ebfac555889..242a9757eb0dbb 100644 --- a/fs/smb/server/connection.h +++ b/fs/smb/server/connection.h @@ -22,6 +22,7 @@ #include "ksmbd_work.h" struct smbdirect_buffer_descriptor_v1; +struct ksmbd_session; #define KSMBD_SOCKET_BACKLOG 16 @@ -124,6 +125,7 @@ struct ksmbd_conn { bool binding; atomic_t refcnt; bool is_aapl; + bool aapl_readdir_attr; /* READDIR_ATTR negotiated */ struct work_struct release_work; }; @@ -166,11 +168,13 @@ extern struct rw_semaphore conn_list_lock; bool ksmbd_conn_alive(struct ksmbd_conn *conn); void ksmbd_conn_wait_idle(struct ksmbd_conn *conn); -int ksmbd_conn_wait_idle_sess_id(struct ksmbd_conn *curr_conn, u64 sess_id); +int ksmbd_conn_wait_idle_sess(struct ksmbd_conn *curr_conn, + struct ksmbd_session *sess); struct ksmbd_conn *ksmbd_conn_alloc(void); void ksmbd_conn_free(struct ksmbd_conn *conn); struct ksmbd_conn *ksmbd_conn_get(struct ksmbd_conn *conn); void ksmbd_conn_put(struct ksmbd_conn *conn); +void ksmbd_conn_abort(struct ksmbd_conn *conn); int ksmbd_conn_wq_init(void); void ksmbd_conn_wq_destroy(void); bool ksmbd_conn_lookup_dialect(struct ksmbd_conn *c); @@ -279,5 +283,5 @@ static inline void ksmbd_conn_set_releasing(struct ksmbd_conn *conn) WRITE_ONCE(conn->status, KSMBD_SESS_RELEASING); } -void ksmbd_all_conn_set_status(u64 sess_id, u32 status); +void ksmbd_all_conn_set_status(struct ksmbd_session *sess, u32 status); #endif /* __CONNECTION_H__ */ diff --git a/fs/smb/server/ksmbd_netlink.h b/fs/smb/server/ksmbd_netlink.h index 8ccd57fd904bc2..af1e760453d9c3 100644 --- a/fs/smb/server/ksmbd_netlink.h +++ b/fs/smb/server/ksmbd_netlink.h @@ -113,7 +113,8 @@ struct ksmbd_startup_request { __u32 max_connections; /* Number of maximum simultaneous connections */ __s8 bind_interfaces_only; __u32 max_ip_connections; /* Number of maximum connection per ip address */ - __s8 reserved[499]; /* Reserved room */ + __s8 aapl_model[32]; /* AAPL model string for Finder icon, e.g. "Xserve" */ + __s8 reserved[467]; /* Reserved room */ __u32 ifc_list_sz; /* interfaces list size */ __s8 ____payload[]; } __packed; @@ -285,6 +286,7 @@ struct ksmbd_spnego_authen_response { * stored in SecurityBuffer of SMB2 SESSION * SETUP response */ + __u64 session_expiry; /* Kerberos ticket expiry time */ __u8 payload[]; /* session key + AP_REP */ }; @@ -377,6 +379,8 @@ enum KSMBD_TREE_CONN_STATUS { #define KSMBD_SHARE_FLAG_UPDATE BIT(14) #define KSMBD_SHARE_FLAG_CROSSMNT BIT(15) #define KSMBD_SHARE_FLAG_CONTINUOUS_AVAILABILITY BIT(16) +#define KSMBD_SHARE_FLAG_HIDE_UNREADABLE BIT(17) +#define KSMBD_SHARE_FLAG_TIME_MACHINE BIT(18) /* * Tree connect request flags. diff --git a/fs/smb/server/ksmbd_work.c b/fs/smb/server/ksmbd_work.c index e2c2f45264beaf..f35335307670fe 100644 --- a/fs/smb/server/ksmbd_work.c +++ b/fs/smb/server/ksmbd_work.c @@ -11,6 +11,7 @@ #include "server.h" #include "connection.h" #include "ksmbd_work.h" +#include "vfs_cache.h" #include "mgmt/ksmbd_ida.h" static struct kmem_cache *work_cache; @@ -56,6 +57,7 @@ struct ksmbd_work *ksmbd_alloc_work_struct(void) INIT_LIST_HEAD(&work->request_entry); INIT_LIST_HEAD(&work->async_request_entry); INIT_LIST_HEAD(&work->fp_entry); + INIT_LIST_HEAD(&work->notify_entry); INIT_LIST_HEAD(&work->aux_read_list); work->iov_alloc_cnt = ARRAY_SIZE(work->iov_inline); work->iov = work->iov_inline; @@ -85,6 +87,9 @@ void ksmbd_free_work_struct(struct ksmbd_work *work) if (work->async_id) ksmbd_release_id(&work->conn->async_ida, work->async_id); + if (work->owns_conn_ref) + ksmbd_conn_put(work->conn); + ksmbd_fd_put(work, work->request_open); kmem_cache_free(work_cache, work); } diff --git a/fs/smb/server/ksmbd_work.h b/fs/smb/server/ksmbd_work.h index 88104f0cf3636c..5f1d3ebab4fb5a 100644 --- a/fs/smb/server/ksmbd_work.h +++ b/fs/smb/server/ksmbd_work.h @@ -12,6 +12,7 @@ struct ksmbd_conn; struct ksmbd_session; struct ksmbd_tree_connect; +struct ksmbd_file; #define KSMBD_WORK_INLINE_IOVS 4 @@ -90,7 +91,11 @@ struct ksmbd_work { bool compress_response:1; /* Is this SYNC or ASYNC ksmbd_work */ bool asynchronous:1; + /* Work owns a reference to @conn. */ + bool owns_conn_ref:1; bool need_invalidate_rkey:1; + bool request_open_chseq_tracked:1; + bool session_setup_reauth:1; unsigned int remote_key; /* cancel works */ @@ -98,12 +103,21 @@ struct ksmbd_work { void **cancel_argv; void (*cancel_fn)(void **argv); + /* + * Refcounted open associated with the SMB2 command currently being + * processed. + */ + struct ksmbd_file *request_open; + __le16 request_open_chseq; + struct work_struct work; /* List head at conn->requests */ struct list_head request_entry; /* List head at conn->async_requests */ struct list_head async_request_entry; struct list_head fp_entry; + /* List head at ksmbd_file->notify_pendings */ + struct list_head notify_entry; }; /** diff --git a/fs/smb/server/mgmt/share_config.c b/fs/smb/server/mgmt/share_config.c index 6f97f8d39657cd..de524df6dd9c6e 100644 --- a/fs/smb/server/mgmt/share_config.c +++ b/fs/smb/server/mgmt/share_config.c @@ -28,6 +28,62 @@ struct ksmbd_veto_pattern { struct list_head list; }; +#ifdef CONFIG_PROC_FS +static const struct ksmbd_const_name ksmbd_share_flag_names[] = { + {KSMBD_SHARE_FLAG_AVAILABLE, "available"}, + {KSMBD_SHARE_FLAG_BROWSEABLE, "browseable"}, + {KSMBD_SHARE_FLAG_WRITEABLE, "writeable"}, + {KSMBD_SHARE_FLAG_READONLY, "read-only"}, + {KSMBD_SHARE_FLAG_GUEST_OK, "guest-ok"}, + {KSMBD_SHARE_FLAG_GUEST_ONLY, "guest-only"}, + {KSMBD_SHARE_FLAG_STORE_DOS_ATTRS, "store-dos-attrs"}, + {KSMBD_SHARE_FLAG_OPLOCKS, "oplocks"}, + {KSMBD_SHARE_FLAG_PIPE, "pipe"}, + {KSMBD_SHARE_FLAG_HIDE_DOT_FILES, "hide-dot-files"}, + {KSMBD_SHARE_FLAG_INHERIT_OWNER, "inherit-owner"}, + {KSMBD_SHARE_FLAG_STREAMS, "streams"}, + {KSMBD_SHARE_FLAG_FOLLOW_SYMLINKS, "follow-symlinks"}, + {KSMBD_SHARE_FLAG_ACL_XATTR, "acl-xattr"}, + {KSMBD_SHARE_FLAG_UPDATE, "update"}, + {KSMBD_SHARE_FLAG_CROSSMNT, "crossmnt"}, + {KSMBD_SHARE_FLAG_CONTINUOUS_AVAILABILITY, "continuous-availability"}, +}; + +static int proc_show_shares(struct seq_file *m, void *v) +{ + struct ksmbd_share_config *share; + int i; + + down_read(&shares_table_lock); + hash_for_each(shares_table, i, share, hlist) { + seq_printf(m, "name:\t%s\n", share->name); + seq_printf(m, "type:\t%s\n", + test_share_config_flag(share, KSMBD_SHARE_FLAG_PIPE) ? + "pipe" : "disk"); + seq_printf(m, "tree_connects:\t%d\n", + atomic_read(&share->tree_connections)); + seq_printf(m, "file_mask:\t0%07o\n", share->create_mask); + seq_printf(m, "directory_mask:\t0%07o\n", share->directory_mask); + seq_puts(m, "flags:\t"); + ksmbd_proc_show_flag_names(m, ksmbd_share_flag_names, + ARRAY_SIZE(ksmbd_share_flag_names), + share->flags); + seq_puts(m, "\n\n"); + } + up_read(&shares_table_lock); + return 0; +} + +int create_proc_shares(void) +{ + if (!ksmbd_proc_create("shares", proc_show_shares, NULL)) + return -ENOMEM; + return 0; +} +#else +int create_proc_shares(void) { return 0; } +#endif + static unsigned int share_name_hash(const char *name) { return jhash(name, strlen(name), 0); @@ -88,9 +144,9 @@ static struct ksmbd_share_config *__share_lookup(const char *name) static int parse_veto_list(struct ksmbd_share_config *share, char *veto_list, - int veto_list_sz) + size_t veto_list_sz) { - int sz = 0; + size_t sz; if (!veto_list_sz) return 0; @@ -98,7 +154,7 @@ static int parse_veto_list(struct ksmbd_share_config *share, while (veto_list_sz > 0) { struct ksmbd_veto_pattern *p; - sz = strlen(veto_list); + sz = strnlen(veto_list, veto_list_sz); if (!sz) break; @@ -106,7 +162,7 @@ static int parse_veto_list(struct ksmbd_share_config *share, if (!p) return -ENOMEM; - p->pattern = kstrdup(veto_list, KSMBD_DEFAULT_GFP); + p->pattern = kstrndup(veto_list, sz, KSMBD_DEFAULT_GFP); if (!p->pattern) { kfree(p); return -ENOMEM; @@ -114,6 +170,9 @@ static int parse_veto_list(struct ksmbd_share_config *share, list_add(&p->list, &share->veto_list); + if (sz == veto_list_sz) + break; + veto_list += sz + 1; veto_list_sz -= (sz + 1); } @@ -156,21 +215,38 @@ static struct ksmbd_share_config *share_config_request(struct ksmbd_work *work, share->flags = resp->flags; atomic_set(&share->refcount, 1); + ksmbd_share_tree_conn_init(share); INIT_LIST_HEAD(&share->veto_list); share->name = kstrdup(name, KSMBD_DEFAULT_GFP); + if (!share->name) { + kill_share(share); + share = NULL; + goto out; + } if (!test_share_config_flag(share, KSMBD_SHARE_FLAG_PIPE)) { - int path_len = PATH_MAX; - - if (resp->payload_sz) - path_len = resp->payload_sz - resp->veto_list_sz; + size_t path_len; - share->path = kstrndup(ksmbd_share_config_path(resp), path_len, - KSMBD_DEFAULT_GFP); - if (!share->path) { - ret = -ENOMEM; + if (resp->payload_sz <= resp->veto_list_sz) { + ret = -EINVAL; } else { - ret = 0; + path_len = resp->payload_sz - resp->veto_list_sz; + if (resp->veto_list_sz) + path_len--; + + if (!path_len) { + ret = -EINVAL; + } else { + share->path = kstrndup( + ksmbd_share_config_path(resp), + path_len, KSMBD_DEFAULT_GFP); + if (!share->path) + ret = -ENOMEM; + else + ret = 0; + } + } + if (share->path) { share->path_sz = strlen(share->path); while (share->path_sz > 1 && share->path[share->path_sz - 1] == '/') @@ -203,7 +279,7 @@ static struct ksmbd_share_config *share_config_request(struct ksmbd_work *work, share->path = NULL; } } - if (ret || !share->name) { + if (ret) { kill_share(share); share = NULL; goto out; diff --git a/fs/smb/server/mgmt/share_config.h b/fs/smb/server/mgmt/share_config.h index d4ac2dd4de2040..d157545fe7d1d1 100644 --- a/fs/smb/server/mgmt/share_config.h +++ b/fs/smb/server/mgmt/share_config.h @@ -24,6 +24,9 @@ struct ksmbd_share_config { struct path vfs_path; atomic_t refcount; +#ifdef CONFIG_PROC_FS + atomic_t tree_connections; +#endif struct hlist_node hlist; unsigned short create_mask; unsigned short directory_mask; @@ -60,6 +63,27 @@ static inline int test_share_config_flag(struct ksmbd_share_config *share, return share->flags & flag; } +#ifdef CONFIG_PROC_FS +static inline void ksmbd_share_tree_conn_init(struct ksmbd_share_config *share) +{ + atomic_set(&share->tree_connections, 0); +} + +static inline void ksmbd_share_tree_conn_inc(struct ksmbd_share_config *share) +{ + atomic_inc(&share->tree_connections); +} + +static inline void ksmbd_share_tree_conn_dec(struct ksmbd_share_config *share) +{ + atomic_dec(&share->tree_connections); +} +#else +static inline void ksmbd_share_tree_conn_init(struct ksmbd_share_config *share) {} +static inline void ksmbd_share_tree_conn_inc(struct ksmbd_share_config *share) {} +static inline void ksmbd_share_tree_conn_dec(struct ksmbd_share_config *share) {} +#endif + void ksmbd_share_config_del(struct ksmbd_share_config *share); void __ksmbd_share_config_put(struct ksmbd_share_config *share); @@ -74,4 +98,5 @@ struct ksmbd_share_config *ksmbd_share_config_get(struct ksmbd_work *work, const char *name); bool ksmbd_share_veto_filename(struct ksmbd_share_config *share, const char *filename); +int create_proc_shares(void); #endif /* __SHARE_CONFIG_MANAGEMENT_H__ */ diff --git a/fs/smb/server/mgmt/tree_connect.c b/fs/smb/server/mgmt/tree_connect.c index 58e5b8592da46f..dd1db3554caedd 100644 --- a/fs/smb/server/mgmt/tree_connect.c +++ b/fs/smb/server/mgmt/tree_connect.c @@ -82,12 +82,15 @@ ksmbd_tree_conn_connect(struct ksmbd_work *work, const char *share_name) down_write(&sess->tree_conns_lock); ret = xa_err(xa_store(&sess->tree_conns, tree_conn->id, tree_conn, KSMBD_DEFAULT_GFP)); + if (!ret) + atomic_inc(&tree_conn->refcount); up_write(&sess->tree_conns_lock); if (ret) { status.ret = -ENOMEM; goto out_error; } ksmbd_counter_inc(KSMBD_COUNTER_TREE_CONNS); + ksmbd_share_tree_conn_inc(sc); kvfree(resp); return status; @@ -116,6 +119,7 @@ static int __ksmbd_tree_conn_disconnect(struct ksmbd_session *sess, ret = ksmbd_ipc_tree_disconnect_request(sess->id, tree_conn->id); ksmbd_release_tree_conn_id(sess, tree_conn->id); ksmbd_counter_dec(KSMBD_COUNTER_TREE_CONNS); + ksmbd_share_tree_conn_dec(tree_conn->share_conf); if (atomic_dec_and_test(&tree_conn->refcount)) { ksmbd_share_config_put(tree_conn->share_conf); kfree(tree_conn); @@ -127,6 +131,12 @@ int ksmbd_tree_conn_disconnect(struct ksmbd_session *sess, struct ksmbd_tree_connect *tree_conn) { down_write(&sess->tree_conns_lock); + if (tree_conn->t_state == TREE_DISCONNECTED || + xa_load(&sess->tree_conns, tree_conn->id) != tree_conn) { + up_write(&sess->tree_conns_lock); + return -ENOENT; + } + tree_conn->t_state = TREE_DISCONNECTED; xa_erase(&sess->tree_conns, tree_conn->id); up_write(&sess->tree_conns_lock); diff --git a/fs/smb/server/mgmt/user_config.c b/fs/smb/server/mgmt/user_config.c index cf45841d9d1b99..0be08cf1896c49 100644 --- a/fs/smb/server/mgmt/user_config.c +++ b/fs/smb/server/mgmt/user_config.c @@ -26,6 +26,7 @@ struct ksmbd_user *ksmbd_login_user(const char *account) resp_ext = ksmbd_ipc_login_request_ext(account); user = ksmbd_alloc_user(resp, resp_ext); + kvfree(resp_ext); out: kvfree(resp); return user; @@ -36,6 +37,17 @@ struct ksmbd_user *ksmbd_alloc_user(struct ksmbd_login_response *resp, { struct ksmbd_user *user; + /* + * resp->hash_sz is a __u16 taken from the mountd IPC login response but + * resp->hash[] is only KSMBD_REQ_MAX_HASH_SZ bytes. A malformed or + * malicious response can set hash_sz far beyond that (up to 65535), + * making the memcpy() below read past the response object + * (slab-out-of-bounds in ksmbd_alloc_user()). Reject any oversized + * hash rather than trust the length. + */ + if (resp->hash_sz > sizeof(resp->hash)) + return NULL; + user = kmalloc_obj(struct ksmbd_user, KSMBD_DEFAULT_GFP); if (!user) return NULL; diff --git a/fs/smb/server/mgmt/user_session.c b/fs/smb/server/mgmt/user_session.c index f99c86284ba3d0..d91dde3f9e6197 100644 --- a/fs/smb/server/mgmt/user_session.c +++ b/fs/smb/server/mgmt/user_session.c @@ -188,6 +188,8 @@ static int create_proc_session(struct ksmbd_session *sess) snprintf(name, sizeof(name), "sessions/%llu", sess->id); sess->proc_entry = ksmbd_proc_create(name, show_proc_session, sess); + if (!sess->proc_entry) + return -ENOMEM; return 0; } @@ -395,6 +397,12 @@ void ksmbd_session_destroy(struct ksmbd_session *sess) kfree(sess); } +static void ksmbd_session_remove_from_table(struct ksmbd_session *sess) +{ + hash_del(&sess->hlist); + ksmbd_counter_dec(KSMBD_COUNTER_SESSIONS); +} + struct ksmbd_session *__session_lookup(unsigned long long id) { struct ksmbd_session *sess; @@ -421,7 +429,7 @@ static void ksmbd_expire_session(struct ksmbd_conn *conn) time_after(jiffies, sess->last_active + SMB2_SESSION_TIMEOUT))) { xa_erase(&conn->sessions, sess->id); - hash_del(&sess->hlist); + ksmbd_session_remove_from_table(sess); ksmbd_session_destroy(sess); continue; } @@ -433,10 +441,21 @@ static void ksmbd_expire_session(struct ksmbd_conn *conn) int ksmbd_session_register(struct ksmbd_conn *conn, struct ksmbd_session *sess) { + int ret; + sess->dialect = conn->dialect; memcpy(sess->ClientGUID, conn->ClientGUID, SMB2_CLIENT_GUID_SIZE); ksmbd_expire_session(conn); - return xa_err(xa_store(&conn->sessions, sess->id, sess, KSMBD_DEFAULT_GFP)); + ret = xa_err(xa_store(&conn->sessions, sess->id, sess, + KSMBD_DEFAULT_GFP)); + if (ret) { + down_write(&sessions_table_lock); + ksmbd_session_remove_from_table(sess); + up_write(&sessions_table_lock); + ksmbd_user_session_put(sess); + } + + return ret; } static int ksmbd_chann_del(struct ksmbd_conn *conn, struct ksmbd_session *sess) @@ -464,7 +483,7 @@ void ksmbd_sessions_deregister(struct ksmbd_conn *conn) hash_for_each_safe(sessions_table, bkt, tmp, sess, hlist) { if (!ksmbd_chann_del(conn, sess) && xa_empty(&sess->ksmbd_chann_list)) { - hash_del(&sess->hlist); + ksmbd_session_remove_from_table(sess); down_write(&conn->session_lock); xa_erase(&conn->sessions, sess->id); up_write(&conn->session_lock); @@ -475,18 +494,10 @@ void ksmbd_sessions_deregister(struct ksmbd_conn *conn) down_write(&conn->session_lock); xa_for_each(&conn->sessions, id, sess) { - unsigned long chann_id; - struct channel *chann; - - xa_for_each(&sess->ksmbd_chann_list, chann_id, chann) { - if (chann->conn != conn) - ksmbd_conn_set_exiting(chann->conn); - } - ksmbd_chann_del(conn, sess); if (xa_empty(&sess->ksmbd_chann_list)) { xa_erase(&conn->sessions, sess->id); - hash_del(&sess->hlist); + ksmbd_session_remove_from_table(sess); if (atomic_dec_and_test(&sess->refcnt)) ksmbd_session_destroy(sess); } @@ -539,19 +550,35 @@ struct ksmbd_session *ksmbd_session_lookup_slowpath(unsigned long long id) return sess; } -struct ksmbd_session *ksmbd_session_lookup_all(struct ksmbd_conn *conn, - unsigned long long id) +struct ksmbd_session *ksmbd_session_lookup_all_states(struct ksmbd_conn *conn, + unsigned long long id) { struct ksmbd_session *sess; + bool channel_found; sess = ksmbd_session_lookup(conn, id); - if (!sess && conn->binding) { + if (!sess) { sess = ksmbd_session_lookup_slowpath(id); - if (sess && !xa_load(&sess->ksmbd_chann_list, (long)conn)) { + if (!sess) + return NULL; + + down_read(&sess->chann_lock); + channel_found = xa_load(&sess->ksmbd_chann_list, (long)conn); + up_read(&sess->chann_lock); + if (!channel_found) { ksmbd_user_session_put(sess); sess = NULL; } } + return sess; +} + +struct ksmbd_session *ksmbd_session_lookup_all(struct ksmbd_conn *conn, + unsigned long long id) +{ + struct ksmbd_session *sess; + + sess = ksmbd_session_lookup_all_states(conn, id); if (sess && sess->state != SMB2_SESSION_VALID) { ksmbd_user_session_put(sess); sess = NULL; @@ -592,6 +619,17 @@ struct preauth_session *ksmbd_preauth_session_alloc(struct ksmbd_conn *conn, return sess; } +void ksmbd_preauth_session_destroy(struct ksmbd_conn *conn) +{ + struct preauth_session *sess, *tmp; + + list_for_each_entry_safe(sess, tmp, &conn->preauth_sess_table, + preauth_entry) { + list_del(&sess->preauth_entry); + kfree(sess); + } +} + void destroy_previous_session(struct ksmbd_conn *conn, struct ksmbd_user *user, u64 id) { @@ -612,16 +650,17 @@ void destroy_previous_session(struct ksmbd_conn *conn, memcmp(user->passkey, prev_user->passkey, user->passkey_sz)) goto out; - ksmbd_all_conn_set_status(id, KSMBD_SESS_NEED_RECONNECT); - err = ksmbd_conn_wait_idle_sess_id(conn, id); + ksmbd_all_conn_set_status(prev_sess, KSMBD_SESS_NEED_RECONNECT); + err = ksmbd_conn_wait_idle_sess(conn, prev_sess); if (err) { - ksmbd_all_conn_set_status(id, KSMBD_SESS_NEED_SETUP); + ksmbd_all_conn_set_status(prev_sess, KSMBD_SESS_NEED_SETUP); goto out; } ksmbd_destroy_file_table(prev_sess); + prev_sess->kerberos_expiry = 0; prev_sess->state = SMB2_SESSION_EXPIRED; - ksmbd_all_conn_set_status(id, KSMBD_SESS_NEED_SETUP); + ksmbd_all_conn_set_status(prev_sess, KSMBD_SESS_NEED_SETUP); ksmbd_launch_ksmbd_durable_scavenger(); out: up_write(&conn->session_lock); @@ -691,10 +730,11 @@ static struct ksmbd_session *__session_create(int protocol) down_write(&sessions_table_lock); hash_add(sessions_table, &sess->hlist, sess->id); + ksmbd_counter_inc(KSMBD_COUNTER_SESSIONS); up_write(&sessions_table_lock); - create_proc_session(sess); - ksmbd_counter_inc(KSMBD_COUNTER_SESSIONS); + if (create_proc_session(sess)) + pr_warn_ratelimited("Unable to create session %llu procfs entry\n", sess->id); return sess; error: diff --git a/fs/smb/server/mgmt/user_session.h b/fs/smb/server/mgmt/user_session.h index 4637a8c8436d02..f8a24c33f7fe43 100644 --- a/fs/smb/server/mgmt/user_session.h +++ b/fs/smb/server/mgmt/user_session.h @@ -47,6 +47,7 @@ struct ksmbd_session { __u8 *Preauth_HashValue; char sess_key[CIFS_KEY_SIZE]; + u64 kerberos_expiry; struct hlist_node hlist; struct rw_semaphore chann_lock; @@ -100,10 +101,13 @@ void ksmbd_sessions_deregister(struct ksmbd_conn *conn); struct ksmbd_session *__session_lookup(unsigned long long id); struct ksmbd_session *ksmbd_session_lookup_all(struct ksmbd_conn *conn, unsigned long long id); +struct ksmbd_session *ksmbd_session_lookup_all_states(struct ksmbd_conn *conn, + unsigned long long id); void destroy_previous_session(struct ksmbd_conn *conn, struct ksmbd_user *user, u64 id); struct preauth_session *ksmbd_preauth_session_alloc(struct ksmbd_conn *conn, u64 sess_id); +void ksmbd_preauth_session_destroy(struct ksmbd_conn *conn); struct preauth_session *ksmbd_preauth_session_lookup(struct ksmbd_conn *conn, unsigned long long id); diff --git a/fs/smb/server/misc.h b/fs/smb/server/misc.h index 3909104e18ad48..c7b063f571a7a6 100644 --- a/fs/smb/server/misc.h +++ b/fs/smb/server/misc.h @@ -43,7 +43,7 @@ struct ksmbd_const_name { const char *name; }; -void ksmbd_proc_init(void); +int ksmbd_proc_init(void); void ksmbd_proc_cleanup(void); void ksmbd_proc_reset(void); struct proc_dir_entry *ksmbd_proc_create(const char *name, @@ -59,7 +59,7 @@ void ksmbd_proc_show_const_name(struct seq_file *m, int count, unsigned int const_value); #else -static inline void ksmbd_proc_init(void) {} +static inline int ksmbd_proc_init(void) { return 0; } static inline void ksmbd_proc_cleanup(void) {} static inline void ksmbd_proc_reset(void) {} #endif diff --git a/fs/smb/server/oplock.c b/fs/smb/server/oplock.c index 32f2f8b219ba66..04dc572ed1bbf2 100644 --- a/fs/smb/server/oplock.c +++ b/fs/smb/server/oplock.c @@ -16,6 +16,7 @@ #include "mgmt/user_session.h" #include "mgmt/share_config.h" #include "mgmt/tree_connect.h" +#include "server.h" static LIST_HEAD(lease_table_list); static DEFINE_RWLOCK(lease_list_lock); @@ -89,6 +90,7 @@ static struct oplock_info *alloc_opinfo(struct ksmbd_work *work, opinfo->conn = ksmbd_conn_get(work->conn); opinfo->level = SMB2_OPLOCK_LEVEL_NONE; opinfo->op_state = OPLOCK_STATE_NONE; + spin_lock_init(&opinfo->state_lock); opinfo->pending_break = 0; opinfo->fid = id; opinfo->Tid = Tid; @@ -545,14 +547,23 @@ void close_id_del_oplock(struct ksmbd_file *fp) opinfo_del(opinfo); rcu_assign_pointer(fp->f_opinfo, NULL); - if (opinfo->op_state == OPLOCK_ACK_WAIT) { - opinfo->op_state = OPLOCK_CLOSING; - wake_up_interruptible_all(&opinfo->oplock_q); - if (opinfo->is_lease) { - atomic_set(&opinfo->breaking_cnt, 0); - wake_up_interruptible_all(&opinfo->oplock_brk); - } - } + spin_lock(&opinfo->state_lock); + if (opinfo->op_state == OPLOCK_ACK_WAIT && opinfo->is_lease) + atomic_set(&opinfo->breaking_cnt, 0); + /* + * An opinfo that has been removed from the inode list is terminal. Keep + * this transition and releasing pending_break under state_lock. a breaker + * takes the same lock before it acquires pending_break or sets ACK_WAIT. + */ + opinfo->op_state = OPLOCK_CLOSING; + clear_bit_unlock(0, &opinfo->pending_break); + spin_unlock(&opinfo->state_lock); + wake_up_interruptible_all(&opinfo->oplock_q); + if (opinfo->is_lease) + wake_up_interruptible_all(&opinfo->oplock_brk); + /* memory barrier is needed for wake_up_bit() */ + smp_mb__after_atomic(); + wake_up_bit(&opinfo->pending_break, 0); opinfo_count_dec(fp); atomic_dec(&opinfo->refcount); @@ -734,12 +745,18 @@ static bool wait_for_break_ack(struct oplock_info *opinfo) /* is this a timeout ? */ if (!rc) { + spin_lock(&opinfo->state_lock); + if (opinfo->op_state == OPLOCK_CLOSING) { + spin_unlock(&opinfo->state_lock); + return false; + } if (opinfo->is_lease) { opinfo->o_lease->state = SMB2_LEASE_NONE_LE; lease_update_oplock_levels(opinfo->o_lease); } opinfo->level = SMB2_OPLOCK_LEVEL_NONE; opinfo->op_state = OPLOCK_STATE_NONE; + spin_unlock(&opinfo->state_lock); return true; } @@ -754,9 +771,35 @@ static void wake_up_oplock_break(struct oplock_info *opinfo) wake_up_bit(&opinfo->pending_break, 0); } +static bool oplock_break_set_ack_wait(struct oplock_info *opinfo) +{ + bool ret = false; + + spin_lock(&opinfo->state_lock); + if (opinfo->op_state != OPLOCK_CLOSING) { + opinfo->op_state = OPLOCK_ACK_WAIT; + ret = true; + } + spin_unlock(&opinfo->state_lock); + + return ret; +} + static int oplock_break_pending(struct oplock_info *opinfo, int req_op_level) { - while (test_and_set_bit(0, &opinfo->pending_break)) { + for (;;) { + bool closing; + + spin_lock(&opinfo->state_lock); + closing = opinfo->op_state == OPLOCK_CLOSING; + if (!closing && !test_and_set_bit(0, &opinfo->pending_break)) { + spin_unlock(&opinfo->state_lock); + break; + } + spin_unlock(&opinfo->state_lock); + if (closing) + return -ENOENT; + if (opinfo->is_lease) opinfo->o_lease->reuse_epoch = true; @@ -765,9 +808,12 @@ static int oplock_break_pending(struct oplock_info *opinfo, int req_op_level) /* Not immediately break to none. */ opinfo->open_trunc = 0; - if (opinfo->op_state == OPLOCK_CLOSING) + spin_lock(&opinfo->state_lock); + closing = opinfo->op_state == OPLOCK_CLOSING; + spin_unlock(&opinfo->state_lock); + if (closing) return -ENOENT; - else if (opinfo->level <= req_op_level) { + if (opinfo->level <= req_op_level) { if (opinfo->is_lease == false) return 1; @@ -1184,7 +1230,11 @@ again: if (lease->state & (SMB2_LEASE_WRITE_CACHING_LE | SMB2_LEASE_HANDLE_CACHING_LE)) { - brk_opinfo->op_state = OPLOCK_ACK_WAIT; + if (!oplock_break_set_ack_wait(brk_opinfo)) { + atomic_dec_if_positive(&brk_opinfo->breaking_cnt); + wake_up_oplock_break(brk_opinfo); + return -ENOENT; + } } else atomic_dec(&brk_opinfo->breaking_cnt); @@ -1231,8 +1281,24 @@ again: return err < 0 ? err : 0; if (brk_opinfo->level == SMB2_OPLOCK_LEVEL_BATCH || - brk_opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE) - brk_opinfo->op_state = OPLOCK_ACK_WAIT; + brk_opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE) { + if (!oplock_break_set_ack_wait(brk_opinfo)) { + wake_up_oplock_break(brk_opinfo); + return -ENOENT; + } + } + + /* + * Keep a conflicting CREATE asynchronous while waiting for an + * oplock-break acknowledgement. Besides avoiding a blocked client + * request, this lets a replay arrive while the original CREATE is + * still pending and be rejected with FILE_NOT_AVAILABLE. + */ + if (in_work) { + setup_async_work(in_work, NULL, NULL); + smb2_send_interim_resp(in_work, STATUS_PENDING); + release_async_work(in_work); + } } err = smb2_oplock_break_noti(brk_opinfo, ci); @@ -1475,12 +1541,13 @@ void smb_lazy_parent_lease_break_close(struct ksmbd_file *fp) * @tid: Tree id of connection * @lctx: lease context information on file open * @share_ret: share mode + * @replay: whether this is a replayed CREATE request * * Return: 0 on success, otherwise error */ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, struct ksmbd_file *fp, __u16 tid, - struct lease_ctx_info *lctx, int share_ret) + struct lease_ctx_info *lctx, int share_ret, bool replay) { int err = 0; int break_level = SMB2_OPLOCK_LEVEL_II; @@ -1564,6 +1631,21 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, prev_op_has_lease = prev_opinfo->is_lease; if (prev_op_has_lease) prev_op_state = prev_opinfo->o_lease->state; + /* + * A replay received while this open is waiting for an oplock or lease + * break must not observe an intermediate level and proceed as a new + * open. This check has to precede break_needed. an oplock may already + * have been downgraded from Batch to II while its acknowledgement is + * still pending. + */ + if (replay && + (test_bit(0, &prev_opinfo->pending_break) || + prev_opinfo->op_state == OPLOCK_ACK_WAIT)) { + err = -EINPROGRESS; + opinfo_put(prev_opinfo); + goto err_out; + } + if (share_ret < 0 && prev_opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE) { err = share_ret; @@ -1594,7 +1676,14 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, goto set_lev; } if (err == -ENOENT) { - if (req_op_level != SMB2_OPLOCK_LEVEL_NONE) + /* + * A pending durable CREATE can lose the previous oplock when + * its holder closes the file. In that case grant the original + * request its full caching state. Other opens still need the + * normal shared-open downgrade below. + */ + if (!prev_durable_open && + req_op_level != SMB2_OPLOCK_LEVEL_NONE) req_op_level = SMB2_OPLOCK_LEVEL_II; goto set_lev; } @@ -1764,7 +1853,10 @@ next: if (!brk_op->is_lease && !send_oplock_break) { brk_op->level = SMB2_OPLOCK_LEVEL_NONE; - brk_op->op_state = OPLOCK_STATE_NONE; + spin_lock(&brk_op->state_lock); + if (brk_op->op_state != OPLOCK_CLOSING) + brk_op->op_state = OPLOCK_STATE_NONE; + spin_unlock(&brk_op->state_lock); } else { oplock_break(brk_op, ci, brk_op->is_lease && !is_trunc ? @@ -2177,6 +2269,82 @@ void create_posix_rsp_buf(char *cc, struct ksmbd_file *fp) SIDUNIX_GROUP, (struct smb_sid *)&buf->SidBuffer[28]); } +/** + * create_aapl_rsp_buf() - build AAPL kAAPL_SERVER_QUERY response + * @cc: buffer to write the create context into (AAPL_RSP_MAX_SIZE bytes) + * @vol_caps: volume capability flags (SMB2_CRTCTX_AAPL_* volume bits) + * @req_bitmap: the client's request bitmap, echoed back in reply_bitmap + * + * Response format follows the layout observed from macOS's own smbd, and + * matches the client-side parsing in AAPL's published public client kernel + * source (public client behavior reference, kAAPL_SERVER_QUERY + * case): reply_bitmap, then server_caps/vol_caps/model-info fields present + * only when their reply_bitmap bit is set: + * reply_bitmap = req_bitmap masked to the fields we support + * server_caps = AAPL_SERVER_CAPS_KSMBD when requested + * vol_caps = caller-supplied + * model string = server_conf.aapl_model (default "Xserve") in UTF-16LE, + * when SMB2_CRTCTX_AAPL_MODEL_INFO requested + * + * Sending reply_bitmap with MODEL_INFO set but no model string causes + * smbfs.kext to enter a broken disconnect path requiring a macOS reboot. + */ +void create_aapl_rsp_buf(char *cc, __u64 vol_caps, __u64 req_bitmap) +{ + struct create_aapl_rsp *buf; + u64 reply_bitmap; + u32 data_len; + + buf = (struct create_aapl_rsp *)cc; + memset(buf, 0, AAPL_RSP_MAX_SIZE); + + reply_bitmap = req_bitmap & (SMB2_CRTCTX_AAPL_SERVER_CAPS | + SMB2_CRTCTX_AAPL_VOLUME_CAPS | + SMB2_CRTCTX_AAPL_MODEL_INFO); + + /* base data: cmd(4)+reserved(4)+reply_bitmap(8)+server_caps(8)+vol_caps(8) */ + data_len = 32; + if (reply_bitmap & SMB2_CRTCTX_AAPL_MODEL_INFO) + data_len += 4 + 4 + AAPL_MODEL_UTF16_BYTES; /* pad2+model_bytes+string */ + + buf->ccontext.DataOffset = cpu_to_le16(offsetof(struct create_aapl_rsp, cmd)); + buf->ccontext.DataLength = cpu_to_le32(data_len); + buf->ccontext.NameOffset = cpu_to_le16(offsetof(struct create_aapl_rsp, Name)); + buf->ccontext.NameLength = cpu_to_le16(SMB2_CREATE_AAPL_LEN); + buf->Name[0] = 'A'; + buf->Name[1] = 'A'; + buf->Name[2] = 'P'; + buf->Name[3] = 'L'; + + buf->cmd = cpu_to_le32(SMB2_CRTCTX_AAPL_SERVER_QUERY); + buf->reply_bitmap = cpu_to_le64(reply_bitmap); + buf->server_caps = (reply_bitmap & SMB2_CRTCTX_AAPL_SERVER_CAPS) ? + cpu_to_le64(AAPL_SERVER_CAPS_KSMBD) : 0; + buf->vol_caps = (reply_bitmap & SMB2_CRTCTX_AAPL_VOLUME_CAPS) ? + cpu_to_le64(vol_caps) : 0; + + if (reply_bitmap & SMB2_CRTCTX_AAPL_MODEL_INFO) { + __le32 *p = (__le32 *)((u8 *)buf + sizeof(*buf)); + __le16 *model_str = (__le16 *)(p + 2); + const char *src = server_conf.aapl_model[0] ? + server_conf.aapl_model : "Xserve"; + int i, model_bytes = 0; + + /* Convert ASCII model string to UTF-16LE in-place */ + for (i = 0; src[i] && i < AAPL_MODEL_MAX_CHARS; i++) { + model_str[i] = cpu_to_le16((unsigned char)src[i]); + model_bytes += 2; + } + + p[0] = 0; /* pad2 */ + p[1] = cpu_to_le32(model_bytes); + + /* Update DataLength to reflect actual model string size */ + buf->ccontext.DataLength = + cpu_to_le32(data_len - AAPL_MODEL_UTF16_BYTES + model_bytes); + } +} + /* * Find lease object(opinfo) for given lease key/fid from lease * break/file close path. diff --git a/fs/smb/server/oplock.h b/fs/smb/server/oplock.h index 3f581d22bb676f..54785df84525e7 100644 --- a/fs/smb/server/oplock.h +++ b/fs/smb/server/oplock.h @@ -66,6 +66,7 @@ struct oplock_info { struct ksmbd_file *o_fp; int level; int op_state; + spinlock_t state_lock; unsigned long pending_break; u64 fid; atomic_t breaking_cnt; @@ -96,7 +97,7 @@ struct oplock_break_info { int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, struct ksmbd_file *fp, __u16 tid, - struct lease_ctx_info *lctx, int share_ret); + struct lease_ctx_info *lctx, int share_ret, bool replay); void smb_break_all_levII_oplock(struct ksmbd_work *work, struct ksmbd_file *fp, int is_trunc); void smb_break_all_levII_oplock_no_interim(struct ksmbd_work *work, @@ -125,6 +126,7 @@ void create_durable_v2_rsp_buf(char *cc, struct ksmbd_file *fp); void create_mxac_rsp_buf(char *cc, int maximal_access); void create_disk_id_rsp_buf(char *cc, __u64 file_id, __u64 vol_id); void create_posix_rsp_buf(char *cc, struct ksmbd_file *fp); +void create_aapl_rsp_buf(char *cc, __u64 vol_caps, __u64 req_bitmap); struct create_context *smb2_find_context_vals(void *open_req, const char *tag, int tag_len); struct oplock_info *lookup_lease_in_table(struct ksmbd_conn *conn, char *lease_key); diff --git a/fs/smb/server/proc.c b/fs/smb/server/proc.c index 101a2cc45a4477..5c9f3f314dbe14 100644 --- a/fs/smb/server/proc.c +++ b/fs/smb/server/proc.c @@ -106,14 +106,14 @@ void ksmbd_proc_reset(void) percpu_counter_set(&ksmbd_counters.counters[i], 0); } -void ksmbd_proc_init(void) +int ksmbd_proc_init(void) { int i; - int retval; + int retval = -ENOMEM; ksmbd_proc_fs = proc_mkdir("fs/ksmbd", NULL); if (!ksmbd_proc_fs) - return; + return retval; if (!proc_mkdir_mode("sessions", 0400, ksmbd_proc_fs)) goto err_out; @@ -124,11 +124,14 @@ void ksmbd_proc_init(void) goto err_out; } - if (!ksmbd_proc_create("server", proc_show_ksmbd_stats, NULL)) + if (!ksmbd_proc_create("server", proc_show_ksmbd_stats, NULL)) { + retval = -ENOMEM; goto err_out; + } ksmbd_proc_reset(); - return; + return 0; err_out: ksmbd_proc_cleanup(); + return retval; } diff --git a/fs/smb/server/server.c b/fs/smb/server/server.c index 960c4c897c11f6..0b080972e90855 100644 --- a/fs/smb/server/server.c +++ b/fs/smb/server/server.c @@ -15,6 +15,7 @@ #include "server.h" #include "smb_common.h" +#include "smb2pdu.h" #include "../common/smb2status.h" #include "connection.h" #include "transport_ipc.h" @@ -182,9 +183,33 @@ static void __handle_ksmbd_work(struct ksmbd_work *work, if (conn->ops->is_transform_hdr && conn->ops->is_transform_hdr(work->request_buf)) { rc = conn->ops->decrypt_req(work); - if (rc < 0) + if (rc < 0) { + ksmbd_conn_abort(conn); return; + } work->encrypted = true; + + /* + * SMB3 applies compression before encryption. The receive loop + * handles a plain compression transform before allocating work, but + * an encrypted request exposes that transform only after decryption. + */ + if (((struct smb2_hdr *)smb_get_msg(work->request_buf))->ProtocolId == + SMB2_COMPRESSION_TRANSFORM_ID) { + rc = ksmbd_decompress_work_request(work); + if (rc < 0) { + ksmbd_conn_abort(conn); + return; + } + } + + /* The decrypted payload must now be a complete SMB2 request. */ + if (((struct smb2_hdr *)smb_get_msg(work->request_buf))->ProtocolId != + SMB2_PROTO_NUMBER || + get_rfc1002_len(work->request_buf) < sizeof(struct smb2_pdu)) { + ksmbd_conn_abort(conn); + return; + } } if (conn->ops->allocate_rsp_buf(work)) @@ -204,6 +229,9 @@ static void __handle_ksmbd_work(struct ksmbd_work *work, if (rc == -EINVAL) conn->ops->set_rsp_status(work, STATUS_INVALID_PARAMETER); + else if (rc == -EKEYEXPIRED) + conn->ops->set_rsp_status(work, + STATUS_NETWORK_SESSION_EXPIRED); else conn->ops->set_rsp_status(work, STATUS_USER_SESSION_DELETED); @@ -211,7 +239,11 @@ static void __handle_ksmbd_work(struct ksmbd_work *work, struct smb2_hdr *rsp_hdr; rsp_hdr = ksmbd_resp_buf_curr(work); - rsp_hdr->Flags |= SMB2_FLAGS_SIGNED; + if (rc == -EKEYEXPIRED && work->sess && + conn->ops->set_sign_rsp) + conn->ops->set_sign_rsp(work); + else + rsp_hdr->Flags |= SMB2_FLAGS_SIGNED; } goto send; } else if (rc > 0) { @@ -229,8 +261,10 @@ static void __handle_ksmbd_work(struct ksmbd_work *work, } rc = __process_request(work, conn, &command); - if (rc == SERVER_HANDLER_ABORT) + if (rc == SERVER_HANDLER_ABORT) { + smb2_complete_request_open(work); break; + } /* * Call smb2_set_rsp_credits() function to set number of credits @@ -243,10 +277,13 @@ static void __handle_ksmbd_work(struct ksmbd_work *work, if (rc < 0) { conn->ops->set_rsp_status(work, STATUS_INVALID_PARAMETER); + smb2_complete_request_open(work); goto send; } } + smb2_complete_request_open(work); + is_chained = is_chained_smb2_message(work); if (work->sess && @@ -262,6 +299,7 @@ static void __handle_ksmbd_work(struct ksmbd_work *work, } while (is_chained == true); send: + smb2_complete_request_open(work); /* * Release any credit charge still outstanding for this request. On * the normal path smb2_set_rsp_credits() already returned it, but the @@ -603,18 +641,25 @@ static int __init ksmbd_server_init(void) return ret; } - ksmbd_proc_init(); - create_proc_sessions(); + ret = ksmbd_proc_init(); + if (ret) + goto err_unregister; + + if (create_proc_sessions()) + pr_warn("Unable to create sessions procfs entry\n"); + + if (create_proc_shares()) + pr_warn("Unable to create shares procfs entry\n"); ksmbd_server_tcp_callbacks_init(); ret = server_conf_init(); if (ret) - goto err_unregister; + goto err_proc_cleanup; ret = ksmbd_work_pool_init(); if (ret) - goto err_unregister; + goto err_proc_cleanup; ret = ksmbd_init_file_cache(); if (ret) @@ -660,6 +705,8 @@ err_exit_file_cache: ksmbd_exit_file_cache(); err_destroy_work_pools: ksmbd_work_pool_destroy(); +err_proc_cleanup: + ksmbd_proc_cleanup(); err_unregister: class_unregister(&ksmbd_control_class); diff --git a/fs/smb/server/server.h b/fs/smb/server/server.h index b8a7317be86b4e..4d4d268b59d5a5 100644 --- a/fs/smb/server/server.h +++ b/fs/smb/server/server.h @@ -48,6 +48,8 @@ struct ksmbd_server_config { char *conf[SERVER_CONF_WORK_GROUP + 1]; struct task_struct *dh_task; bool bind_interfaces_only; + /* AAPL model string for Finder icon, e.g. "Xserve" */ + char aapl_model[32]; }; extern struct ksmbd_server_config server_conf; diff --git a/fs/smb/server/smb2ops.c b/fs/smb/server/smb2ops.c index c9a32ee096b582..8c1f5b5c982873 100644 --- a/fs/smb/server/smb2ops.c +++ b/fs/smb/server/smb2ops.c @@ -37,6 +37,7 @@ static struct smb_version_values smb21_server_values = { .create_mxac_size = sizeof(struct create_mxac_rsp), .create_disk_id_size = sizeof(struct create_disk_id_rsp), .create_posix_size = sizeof(struct create_posix_rsp), + .create_aapl_size = AAPL_RSP_MAX_SIZE, }; static struct smb_version_values smb30_server_values = { @@ -64,6 +65,7 @@ static struct smb_version_values smb30_server_values = { .create_mxac_size = sizeof(struct create_mxac_rsp), .create_disk_id_size = sizeof(struct create_disk_id_rsp), .create_posix_size = sizeof(struct create_posix_rsp), + .create_aapl_size = AAPL_RSP_MAX_SIZE, }; static struct smb_version_values smb302_server_values = { @@ -91,6 +93,7 @@ static struct smb_version_values smb302_server_values = { .create_mxac_size = sizeof(struct create_mxac_rsp), .create_disk_id_size = sizeof(struct create_disk_id_rsp), .create_posix_size = sizeof(struct create_posix_rsp), + .create_aapl_size = AAPL_RSP_MAX_SIZE, }; static struct smb_version_values smb311_server_values = { @@ -118,6 +121,7 @@ static struct smb_version_values smb311_server_values = { .create_mxac_size = sizeof(struct create_mxac_rsp), .create_disk_id_size = sizeof(struct create_disk_id_rsp), .create_posix_size = sizeof(struct create_posix_rsp), + .create_aapl_size = AAPL_RSP_MAX_SIZE, }; static struct smb_version_ops smb2_0_server_ops = { @@ -266,8 +270,10 @@ void init_smb3_02_server(struct ksmbd_conn *conn) if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB3_MULTICHANNEL) conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_MULTI_CHANNEL; - if (server_conf.flags & KSMBD_GLOBAL_FLAG_DURABLE_HANDLE) - conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_PERSISTENT_HANDLES; + /* + * Durable handles are in-memory only. Do not advertise persistent + * handles until CA recovery and fencing are implemented. + */ } /** @@ -290,10 +296,7 @@ int init_smb3_11_server(struct ksmbd_conn *conn) if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB3_MULTICHANNEL) conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_MULTI_CHANNEL; - if (server_conf.flags & KSMBD_GLOBAL_FLAG_DURABLE_HANDLE) - conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_PERSISTENT_HANDLES; - - INIT_LIST_HEAD(&conn->preauth_sess_table); + /* See init_smb3_02_server(): persistent handles require CA recovery. */ return 0; } diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c index e02e254645e202..4cf7083f35ccdf 100644 --- a/fs/smb/server/smb2pdu.c +++ b/fs/smb/server/smb2pdu.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "glob.h" #include "../common/smbfsctl.h" @@ -55,6 +56,10 @@ static void __wbuf(struct ksmbd_work *work, void **req, void **rsp) } } +static struct ksmbd_work *smb2_notify_cancel_claim(void **argv); +static void smb2_notify_cancel_fn(void **argv); +static void smb2_complete_notify_cancel(struct ksmbd_work *in_work); + #define WORK_BUFFERS(w, rq, rs) __wbuf((w), (void **)&(rq), (void **)&(rs)) #define SMB2_CREATE_FILE_ATTRIBUTE_MASK \ @@ -64,28 +69,8 @@ static void __wbuf(struct ksmbd_work *work, void **req, void **rsp) /* Windows reports automatic write-time updates at roughly 15 ms resolution. */ #define KSMBD_WRITE_TIME_RESOLUTION (15ULL * 10000) -/** - * check_session_id() - check for valid session id in smb header - * @conn: connection instance - * @id: session id from smb header - * - * Return: 1 if valid session id, otherwise 0 - */ -static inline bool check_session_id(struct ksmbd_conn *conn, u64 id) -{ - struct ksmbd_session *sess; - - if (id == 0 || id == -1) - return false; - - sess = ksmbd_session_lookup_all(conn, id); - if (sess) { - ksmbd_user_session_put(sess); - return true; - } - pr_err("Invalid user session id: %llu\n", id); - return false; -} +/* MAXFILESIZE in [MS-FSA] 2.1.5.3 Server Requests a Write. */ +#define SMB2_MAX_FILE_SIZE 0xfffffff0000ULL struct channel *lookup_chann_list(struct ksmbd_session *sess, struct ksmbd_conn *conn) { @@ -551,6 +536,8 @@ static void init_chained_smb2_rsp(struct ksmbd_work *work) */ rsp_hdr->Flags = (SMB2_FLAGS_SERVER_TO_REDIR | SMB2_FLAGS_RELATED_OPERATIONS); + if (rcv_hdr->Flags & SMB2_FLAGS_REPLAY_OPERATION) + rsp_hdr->Flags |= SMB2_FLAGS_REPLAY_OPERATION; rsp_hdr->NextCommand = 0; rsp_hdr->MessageId = rcv_hdr->MessageId; rsp_hdr->Id.SyncId.ProcessId = rcv_hdr->Id.SyncId.ProcessId; @@ -643,6 +630,8 @@ int init_smb2_rsp_hdr(struct ksmbd_work *work) * Message is response. We don't grant oplock yet. */ rsp_hdr->Flags = (SMB2_FLAGS_SERVER_TO_REDIR); + if (rcv_hdr->Flags & SMB2_FLAGS_REPLAY_OPERATION) + rsp_hdr->Flags |= SMB2_FLAGS_REPLAY_OPERATION; rsp_hdr->NextCommand = 0; rsp_hdr->MessageId = rcv_hdr->MessageId; rsp_hdr->Id.SyncId.ProcessId = rcv_hdr->Id.SyncId.ProcessId; @@ -653,6 +642,192 @@ int init_smb2_rsp_hdr(struct ksmbd_work *work) return 0; } +static __le16 smb3_hdr_channel_sequence(struct smb2_hdr *hdr) +{ + return ((struct smb3_hdr_req *)hdr)->ChannelSequence; +} + +static bool smb3_hdr_replay(struct smb2_hdr *hdr) +{ + return hdr->Flags & SMB2_FLAGS_REPLAY_OPERATION; +} + +static int smb3_verify_channel_sequence(struct ksmbd_work *work, + struct ksmbd_file *fp, + struct smb2_hdr *hdr, + bool allow_stale) +{ + __le16 chseq_le; + u16 chseq, old_chseq; + int ret = 0; + + if (work->conn->dialect < SMB30_PROT_ID) + return 0; + + chseq_le = smb3_hdr_channel_sequence(hdr); + chseq = le16_to_cpu(chseq_le); + + spin_lock(&fp->f_lock); + old_chseq = le16_to_cpu(fp->channel_sequence); + if (smb3_hdr_replay(hdr)) { + if (chseq == old_chseq && fp->outstanding_pre_requests == 0) { + fp->outstanding_requests++; + } else if ((u16)(chseq - old_chseq) <= 0x7fff && + fp->outstanding_pre_requests == 0) { + fp->outstanding_pre_requests += fp->outstanding_requests; + fp->outstanding_requests = 1; + fp->channel_sequence = chseq_le; + } else if (allow_stale) { + fp->outstanding_pre_requests++; + } else { + ret = -EAGAIN; + } + } else { + if (chseq == old_chseq) { + fp->outstanding_requests++; + } else if ((u16)(chseq - old_chseq) <= 0x7fff) { + fp->outstanding_pre_requests += fp->outstanding_requests; + fp->outstanding_requests = 1; + fp->channel_sequence = chseq_le; + } else if (allow_stale) { + fp->outstanding_pre_requests++; + } else { + ret = -EAGAIN; + } + } + spin_unlock(&fp->f_lock); + + return ret; +} + +static void smb3_complete_channel_sequence(struct ksmbd_work *work, + struct ksmbd_file *fp, + __le16 chseq_le) +{ + u16 chseq; + + if (work->conn->dialect < SMB30_PROT_ID) + return; + + chseq = le16_to_cpu(chseq_le); + + spin_lock(&fp->f_lock); + if (chseq == le16_to_cpu(fp->channel_sequence)) { + if (fp->outstanding_requests) + fp->outstanding_requests--; + } else { + if (fp->outstanding_pre_requests) + fp->outstanding_pre_requests--; + } + spin_unlock(&fp->f_lock); +} + +static int smb2_set_request_open(struct ksmbd_work *work, struct ksmbd_file *fp, + struct smb2_hdr *hdr, bool verify_chseq, + bool allow_stale_chseq) +{ + struct ksmbd_file *open; + int ret; + + smb2_complete_request_open(work); + + open = ksmbd_file_get(fp); + if (!open) + return -ESTALE; + + if (verify_chseq) { + ret = smb3_verify_channel_sequence(work, fp, hdr, + allow_stale_chseq); + if (ret) { + ksmbd_fd_put(work, open); + return ret; + } + work->request_open_chseq_tracked = true; + } + + work->request_open = open; + work->request_open_chseq = smb3_hdr_channel_sequence(hdr); + return 0; +} + +void smb2_complete_request_open(struct ksmbd_work *work) +{ + struct ksmbd_file *open = work->request_open; + + if (!open) + return; + + if (work->request_open_chseq_tracked) + smb3_complete_channel_sequence(work, open, + work->request_open_chseq); + + work->request_open = NULL; + work->request_open_chseq_tracked = false; + ksmbd_fd_put(work, open); +} + +static bool smb2_lock_sequence_applicable(struct ksmbd_work *work, + struct ksmbd_file *fp) +{ + return fp->is_resilient || fp->is_durable || fp->is_persistent || + (work->conn->dialect >= SMB30_PROT_ID && + (work->conn->vals->req_capabilities & + SMB2_GLOBAL_CAP_MULTI_CHANNEL)); +} + +static bool smb2_verify_lock_sequence(struct ksmbd_work *work, + struct ksmbd_file *fp, + struct smb2_lock_req *req) +{ + u32 val, index; + u8 sequence; + bool replay = false; + + if (work->conn->dialect == SMB20_PROT_ID || + !smb2_lock_sequence_applicable(work, fp)) + return false; + + val = le32_to_cpu(req->LockSequenceNumber); + sequence = val & 0xf; + index = val >> 4; + if (!index || index > KSMBD_LOCK_SEQ_ARRAY_SIZE) + return false; + + spin_lock(&fp->f_lock); + if (fp->lock_seq[index - 1].valid) { + if (fp->lock_seq[index - 1].sequence == sequence) + replay = true; + else + fp->lock_seq[index - 1].valid = false; + } + spin_unlock(&fp->f_lock); + + return replay; +} + +static void smb2_update_lock_sequence(struct ksmbd_work *work, + struct ksmbd_file *fp, + struct smb2_lock_req *req) +{ + u32 val, index; + u8 sequence; + + if (work->conn->dialect == SMB20_PROT_ID || + !smb2_lock_sequence_applicable(work, fp)) + return; + + val = le32_to_cpu(req->LockSequenceNumber); + sequence = val & 0xf; + index = val >> 4; + if (!index || index > KSMBD_LOCK_SEQ_ARRAY_SIZE) + return; + + spin_lock(&fp->f_lock); + fp->lock_seq[index - 1].valid = true; + fp->lock_seq[index - 1].sequence = sequence; + spin_unlock(&fp->f_lock); +} + /** * smb2_allocate_rsp_buf() - allocate smb2 response buffer * @work: smb work containing smb request buffer @@ -697,6 +872,47 @@ int smb2_allocate_rsp_buf(struct ksmbd_work *work) return 0; } +static bool smb2_session_expired_cmd_allowed(struct ksmbd_work *work, + unsigned int cmd) +{ + struct smb2_lock_req *req; + unsigned int len, lock_count, i; + + if (cmd == SMB2_CANCEL_HE || cmd == SMB2_CLOSE_HE || + cmd == SMB2_LOGOFF_HE) + return true; + if (cmd != SMB2_LOCK_HE) + return false; + + req = ksmbd_req_buf_next(work); + if (req->hdr.NextCommand) + len = le32_to_cpu(req->hdr.NextCommand); + else { + len = get_rfc1002_len(work->request_buf); + if (len < work->next_smb2_rcv_hdr_off) + return false; + len -= work->next_smb2_rcv_hdr_off; + } + + lock_count = le16_to_cpu(req->LockCount); + if (!lock_count || len < offsetof(struct smb2_lock_req, locks) || + lock_count > (len - offsetof(struct smb2_lock_req, locks)) / + sizeof(struct smb2_lock_element)) + return false; + + for (i = 0; i < lock_count; i++) { + if (le32_to_cpu(req->locks[i].Flags) != SMB2_LOCKFLAG_UNLOCK) + return false; + } + return true; +} + +static bool smb2_session_kerberos_expired(struct ksmbd_session *sess) +{ + return sess->kerberos_expiry && + ktime_get_real_seconds() >= sess->kerberos_expiry; +} + /** * smb2_check_user_session() - check for valid session for a user * @work: smb work containing smb request buffer @@ -711,19 +927,37 @@ int smb2_check_user_session(struct ksmbd_work *work) unsigned long long sess_id; /* - * SMB2_ECHO, SMB2_NEGOTIATE, SMB2_SESSION_SETUP command do not - * require a session id, so no need to validate user session's for - * these commands. + * SMB2_NEGOTIATE and SMB2_SESSION_SETUP do not require a session id. + * SMB2_ECHO may omit it, but an echo carrying a session id still needs + * the session attached to work so that its signature can be checked and + * the response can be signed, including after Kerberos expiry. */ - if (cmd == SMB2_ECHO_HE || cmd == SMB2_NEGOTIATE_HE || - cmd == SMB2_SESSION_SETUP_HE) + if (cmd == SMB2_NEGOTIATE_HE || cmd == SMB2_SESSION_SETUP_HE) + return 0; + + sess_id = le64_to_cpu(req_hdr->SessionId); + if (cmd == SMB2_ECHO_HE) { + /* + * ECHO remains valid without a live session, including after + * LOGOFF. Attach an existing session only to authenticate a signed + * ECHO and sign its response; a stale SessionId is not an error. + */ + if (!work->next_smb2_rcv_hdr_off && sess_id) + work->sess = ksmbd_session_lookup_all_states(conn, sess_id); + if (work->sess) { + if (smb2_session_kerberos_expired(work->sess)) { + work->sess->state = SMB2_SESSION_EXPIRED; + } else if (work->sess->state != SMB2_SESSION_VALID) { + ksmbd_user_session_put(work->sess); + work->sess = NULL; + } + } return 0; + } if (!ksmbd_conn_good(conn)) return -EIO; - sess_id = le64_to_cpu(req_hdr->SessionId); - /* * If request is not the first in Compound request, * Just validate session id in header with work->sess->id. @@ -738,18 +972,43 @@ int smb2_check_user_session(struct ksmbd_work *work) sess_id, work->sess->id); return -EINVAL; } + if (smb2_session_kerberos_expired(work->sess)) + work->sess->state = SMB2_SESSION_EXPIRED; if (work->sess->state != SMB2_SESSION_VALID) { pr_err("compound request on a non-valid session (state %d)\n", work->sess->state); - return -EINVAL; + if (smb2_session_kerberos_expired(work->sess) && + smb2_session_expired_cmd_allowed(work, cmd)) + return 1; + return smb2_session_kerberos_expired(work->sess) ? + -EKEYEXPIRED : -EINVAL; } return 1; } /* Check for validity of user session */ - work->sess = ksmbd_session_lookup_all(conn, sess_id); - if (work->sess) + work->sess = ksmbd_session_lookup_all_states(conn, sess_id); + if (work->sess) { + if (smb2_session_kerberos_expired(work->sess)) { + work->sess->state = SMB2_SESSION_EXPIRED; + return smb2_session_expired_cmd_allowed(work, cmd) ? + 1 : -EKEYEXPIRED; + } + if (work->sess->state != SMB2_SESSION_VALID) { + /* + * Keep the reference for an encrypted request so the caller can + * return STATUS_USER_SESSION_DELETED encrypted with the old key. + */ + if (work->encrypted && + work->sess->state == SMB2_SESSION_EXPIRED && + work->sess->enc) + return -ENOENT; + ksmbd_user_session_put(work->sess); + work->sess = NULL; + return -ENOENT; + } return 1; + } ksmbd_debug(SMB, "Invalid user session, Uid %llu\n", sess_id); return -ENOENT; } @@ -1393,7 +1652,7 @@ int smb2_handle_negotiate(struct ksmbd_work *work) KSMBD_DEFAULT_GFP); if (!conn->preauth_info) { rc = -ENOMEM; - rsp->hdr.Status = STATUS_INVALID_PARAMETER; + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; goto err_out; } @@ -1490,7 +1749,7 @@ int smb2_handle_negotiate(struct ksmbd_work *work) ksmbd_conn_set_need_setup(conn); err_out: - if (rc) + if (rc && rsp->hdr.Status == STATUS_SUCCESS) rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; if (!rc) @@ -1820,7 +2079,9 @@ static int krb5_authenticate(struct ksmbd_work *work, struct ksmbd_session *sess = work->sess; char *in_blob, *out_blob; char channel_key[CIFS_KEY_SIZE] = {}; - char *auth_key = conn->binding ? channel_key : sess->sess_key; + char reauth_key[CIFS_KEY_SIZE] = {}; + char *auth_key = conn->binding ? channel_key : + (work->session_setup_reauth ? reauth_key : sess->sess_key); u64 prev_sess_id; bool binding = conn->binding; int in_len, out_len; @@ -1839,7 +2100,7 @@ static int krb5_authenticate(struct ksmbd_work *work, if (retval) { ksmbd_debug(SMB, "krb5 authentication failed\n"); if (retval != -EKEYREJECTED) - retval = -EINVAL; + retval = -EPERM; goto out; } @@ -1855,12 +2116,21 @@ static int krb5_authenticate(struct ksmbd_work *work, * that it is reauthentication. And the user/password * has been verified, so return it here. */ - if (sess->state == SMB2_SESSION_VALID) { + if (sess->state == SMB2_SESSION_VALID && !work->session_setup_reauth) { if (conn->binding) goto binding_session; return 0; } + /* + * Reauthentication verifies the new Kerberos credentials but keeps + * the established SMB session keys. + */ + if (work->session_setup_reauth) { + retval = 0; + goto out; + } + if ((rsp->SessionFlags != SMB2_SESSION_FLAG_IS_GUEST_LE && (conn->sign || server_conf.enforced_signing)) || (req->SecurityMode & SMB2_NEGOTIATE_SIGNING_REQUIRED)) @@ -1898,6 +2168,7 @@ binding_session: } retval = 0; out: + memzero_explicit(reauth_key, sizeof(reauth_key)); if (binding) memzero_explicit(channel_key, sizeof(channel_key)); return retval; @@ -2041,8 +2312,13 @@ int smb2_sess_setup(struct ksmbd_work *work) } if (sess->state == SMB2_SESSION_EXPIRED) { - rc = -EFAULT; - goto out_err; + if (sess->kerberos_expiry && + ktime_get_real_seconds() >= sess->kerberos_expiry) { + work->session_setup_reauth = true; + } else { + rc = -EFAULT; + goto out_err; + } } if (ksmbd_conn_need_reconnect(conn)) { @@ -2086,10 +2362,8 @@ int smb2_sess_setup(struct ksmbd_work *work) if (conn->preferred_auth_mech & (KSMBD_AUTH_KRB5 | KSMBD_AUTH_MSKRB5)) { rc = krb5_authenticate(work, req, rsp); - if (rc) { - rc = -EINVAL; + if (rc) goto out_err; - } if (!ksmbd_conn_need_reconnect(conn)) { ksmbd_conn_set_good(conn); @@ -2200,6 +2474,7 @@ out_err: */ if (!(req->Flags & SMB2_SESSION_REQ_FLAG_BINDING)) { sess->last_active = jiffies; + sess->kerberos_expiry = 0; sess->state = SMB2_SESSION_EXPIRED; } /* @@ -2251,6 +2526,7 @@ int smb2_tree_connect(struct ksmbd_work *work) struct ksmbd_session *sess = work->sess; char *treename = NULL, *name = NULL; struct ksmbd_tree_conn_status status; + struct ksmbd_tree_connect *tree_conn = NULL; struct ksmbd_share_config *share = NULL; int rc = -EINVAL; @@ -2277,9 +2553,10 @@ int smb2_tree_connect(struct ksmbd_work *work) name, treename); status = ksmbd_tree_conn_connect(work, name); - if (status.ret == KSMBD_TREE_CONN_STATUS_OK) + if (status.ret == KSMBD_TREE_CONN_STATUS_OK) { + tree_conn = status.tree_conn; rsp->hdr.Id.SyncId.TreeId = cpu_to_le32(status.tree_conn->id); - else + } else goto out_err1; share = status.tree_conn->share_conf; @@ -2311,16 +2588,22 @@ int smb2_tree_connect(struct ksmbd_work *work) status.tree_conn->posix_extensions = true; down_write(&sess->tree_conns_lock); - status.tree_conn->t_state = TREE_CONNECTED; + if (status.tree_conn->t_state == TREE_DISCONNECTED) { + status.ret = KSMBD_TREE_CONN_STATUS_ERROR; + share = NULL; + } else { + status.tree_conn->t_state = TREE_CONNECTED; + } up_write(&sess->tree_conns_lock); + if (status.ret != KSMBD_TREE_CONN_STATUS_OK) + goto out_err1; rsp->StructureSize = cpu_to_le16(16); out_err1: - if (server_conf.flags & KSMBD_GLOBAL_FLAG_DURABLE_HANDLE && share && - test_share_config_flag(share, - KSMBD_SHARE_FLAG_CONTINUOUS_AVAILABILITY)) - rsp->Capabilities = SMB2_SHARE_CAP_CONTINUOUS_AVAILABILITY; - else - rsp->Capabilities = 0; + /* + * A configured CA share is not continuously available until persistent + * open recovery, ownership fencing, and failover are implemented. + */ + rsp->Capabilities = 0; rsp->Reserved = 0; /* default manual caching */ rsp->ShareFlags = SMB2_SHAREFLAG_MANUAL_CACHING; @@ -2328,10 +2611,19 @@ out_err1: if (conn->dialect == SMB311_PROT_ID && conn->compress_algorithm != SMB3_COMPRESS_NONE) rsp->ShareFlags |= cpu_to_le32(SMB2_SHAREFLAG_COMPRESS_DATA); + if (share && test_share_config_flag(share, + KSMBD_SHARE_FLAG_HIDE_UNREADABLE)) + rsp->ShareFlags |= + cpu_to_le32(SMB2_SHAREFLAG_ACCESS_BASED_DIRECTORY_ENUM); rc = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_tree_connect_rsp)); - if (rc) + if (rc) { + if (status.ret == KSMBD_TREE_CONN_STATUS_OK) { + ksmbd_tree_conn_disconnect(sess, status.tree_conn); + status.tree_conn = NULL; + } status.ret = KSMBD_TREE_CONN_STATUS_NOMEM; + } if (!IS_ERR(treename)) kfree(treename); @@ -2367,6 +2659,9 @@ out_err1: if (status.ret != KSMBD_TREE_CONN_STATUS_OK) smb2_set_err_rsp(work); + if (tree_conn) + ksmbd_tree_connect_put(tree_conn); + return rc; } @@ -2470,17 +2765,6 @@ int smb2_tree_disconnect(struct ksmbd_work *work) ksmbd_close_tree_conn_fds(work); - down_write(&sess->tree_conns_lock); - if (tcon->t_state == TREE_DISCONNECTED) { - up_write(&sess->tree_conns_lock); - rsp->hdr.Status = STATUS_NETWORK_NAME_DELETED; - err = -ENOENT; - goto err_out; - } - - tcon->t_state = TREE_DISCONNECTED; - up_write(&sess->tree_conns_lock); - err = ksmbd_tree_conn_disconnect(sess, tcon); if (err) { rsp->hdr.Status = STATUS_NETWORK_NAME_DELETED; @@ -2515,7 +2799,6 @@ int smb2_session_logoff(struct ksmbd_work *work) struct ksmbd_session *sess = work->sess; struct smb2_logoff_req *req; struct smb2_logoff_rsp *rsp; - u64 sess_id; int err; WORK_BUFFERS(work, req, rsp); @@ -2529,8 +2812,7 @@ int smb2_session_logoff(struct ksmbd_work *work) smb2_set_err_rsp(work); return -ENOENT; } - sess_id = le64_to_cpu(req->hdr.SessionId); - ksmbd_all_conn_set_status(sess_id, KSMBD_SESS_NEED_RECONNECT); + ksmbd_all_conn_set_status(sess, KSMBD_SESS_NEED_RECONNECT); ksmbd_conn_unlock(conn); ksmbd_close_session_fds(work); @@ -2544,10 +2826,11 @@ int smb2_session_logoff(struct ksmbd_work *work) } down_write(&conn->session_lock); + sess->kerberos_expiry = 0; sess->state = SMB2_SESSION_EXPIRED; up_write(&conn->session_lock); - ksmbd_all_conn_set_status(sess_id, KSMBD_SESS_NEED_SETUP); + ksmbd_all_conn_set_status(sess, KSMBD_SESS_NEED_SETUP); rsp->StructureSize = cpu_to_le16(4); err = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_logoff_rsp)); @@ -2974,12 +3257,86 @@ struct durable_info { unsigned short int type; bool persistent; bool reconnected; + bool replay; + bool replay_consumed; bool app_instance_id; unsigned int timeout; char *CreateGuid; char AppInstanceId[SMB2_CREATE_GUID_SIZE]; }; +static int smb2_check_durable_replay(struct ksmbd_work *work, + struct ksmbd_file *fp, + struct lease_ctx_info *lc, + bool persistent) +{ + struct oplock_info *opinfo; + int ret = 0; + + if (!fp->is_durable && !fp->is_persistent) + return -EACCES; + + if (ksmbd_vfs_compare_durable_owner(fp, work->sess->user) == false) + return -EACCES; + + if (fp->is_persistent && !persistent) + return -EINVAL; + + opinfo = opinfo_get(fp); + if (!opinfo) + return 0; + + if (opinfo->sess && opinfo->sess->id != work->sess->id) { + ret = -ENOEXEC; + goto out; + } + + if (opinfo->is_lease) { + if (!lc || + memcmp(opinfo->o_lease->lease_key, lc->lease_key, + SMB2_LEASE_KEY_SIZE)) { + ret = -EACCES; + goto out; + } + } else { + if (lc) { + ret = -EACCES; + goto out; + } + + if (fp->is_durable && opinfo->level != SMB2_OPLOCK_LEVEL_BATCH) + ret = -EACCES; + } +out: + opinfo_put(opinfo); + return ret; +} + +static bool smb2_durable_replay_consumed(struct ksmbd_file *fp) +{ + bool consumed; + + spin_lock(&fp->f_lock); + consumed = fp->durable_replay_consumed; + spin_unlock(&fp->f_lock); + + return consumed; +} + +static void smb2_mark_durable_replay_consumed(struct ksmbd_file *fp) +{ + spin_lock(&fp->f_lock); + fp->durable_replay_consumed = true; + spin_unlock(&fp->f_lock); +} + +static bool smb2_durable_replay_differs(struct ksmbd_file *fp, + struct smb2_create_req *req) +{ + return fp->cdoption != req->CreateDisposition || + fp->create_file_attributes != req->FileAttributes; +} + static int parse_durable_handle_context(struct ksmbd_work *work, struct smb2_create_req *req, struct lease_ctx_info *lc, @@ -3107,6 +3464,10 @@ static int parse_durable_handle_context(struct ksmbd_work *work, durable_v2_blob = (struct create_durable_req_v2 *)context; ksmbd_debug(SMB, "Request for durable v2 open\n"); + dh_info->CreateGuid = durable_v2_blob->dcontext.CreateGuid; + dh_info->persistent = + le32_to_cpu(durable_v2_blob->dcontext.Flags) & + SMB2_DHANDLE_FLAG_PERSISTENT; dh_info->fp = ksmbd_lookup_fd_cguid(durable_v2_blob->dcontext.CreateGuid); if (dh_info->fp) { if (!memcmp(conn->ClientGUID, dh_info->fp->client_guid, @@ -3117,12 +3478,73 @@ static int parse_durable_handle_context(struct ksmbd_work *work, goto out; } - if (dh_info->fp->conn) { + if (dh_info->fp->f_state == FP_NEW) { + /* Original CREATE is still pending. */ + ksmbd_put_durable_fd(dh_info->fp); + err = -EAGAIN; + goto out; + } + + if (!dh_info->fp->is_durable && + !dh_info->fp->is_persistent) { + /* + * A DurableHandleReqV2 CREATE can complete + * without granting durability (for example, if + * it requested no oplock). Its CreateGuid still + * identifies a completed CREATE for replay. + */ + if (dh_info->fp->conn && + ksmbd_vfs_compare_durable_owner( + dh_info->fp, work->sess->user)) { + if (smb2_durable_replay_consumed( + dh_info->fp)) { + ksmbd_put_durable_fd(dh_info->fp); + dh_info->fp = NULL; + dh_info->type = dh_idx; + dh_info->replay_consumed = true; + break; + } + if (smb2_durable_replay_differs( + dh_info->fp, req)) + smb2_mark_durable_replay_consumed( + dh_info->fp); + dh_info->replay = true; + dh_info->type = dh_idx; + goto out; + } ksmbd_put_durable_fd(dh_info->fp); - err = -EBADF; + err = -EACCES; goto out; } - dh_info->reconnected = true; + + if (dh_info->fp->conn && + smb2_durable_replay_consumed(dh_info->fp)) { + ksmbd_put_durable_fd(dh_info->fp); + dh_info->fp = NULL; + dh_info->type = dh_idx; + dh_info->replay_consumed = true; + break; + } + + err = smb2_check_durable_replay(work, + dh_info->fp, + lc, + dh_info->persistent); + if (err) { + ksmbd_put_durable_fd(dh_info->fp); + goto out; + } + + if (dh_info->fp->conn) { + if (smb2_durable_replay_differs(dh_info->fp, + req)) + smb2_mark_durable_replay_consumed( + dh_info->fp); + dh_info->replay = true; + } else { + dh_info->reconnected = true; + } + dh_info->type = dh_idx; goto out; } ksmbd_put_durable_fd(dh_info->fp); @@ -3131,10 +3553,6 @@ static int parse_durable_handle_context(struct ksmbd_work *work, if ((lc && (lc->req_state & SMB2_LEASE_HANDLE_CACHING_LE)) || req_op_level == SMB2_OPLOCK_LEVEL_BATCH) { - dh_info->CreateGuid = - durable_v2_blob->dcontext.CreateGuid; - dh_info->persistent = - le32_to_cpu(durable_v2_blob->dcontext.Flags); dh_info->timeout = le32_to_cpu(durable_v2_blob->dcontext.Timeout); dh_info->type = dh_idx; @@ -3216,6 +3634,9 @@ int smb2_open(struct ksmbd_work *work) int rc = 0; int contxt_cnt = 0, query_disk_id = 0; bool maximal_access_ctxt = false, posix_ctxt = false; + bool aapl_ctxt = false; + bool durable_rsp = true; + __u64 aapl_req_bitmap = 0, aapl_client_caps = 0; int s_type = 0; int next_off = 0; char *name = NULL; @@ -3333,6 +3754,21 @@ int smb2_open(struct ksmbd_work *work) if (rc) goto err_out2; + if (dh_info.replay == true) { + fp = dh_info.fp; + if (ksmbd_override_fsids(work)) { + rc = -ENOMEM; + goto err_out2; + } + + file_info = FILE_OPENED; + rc = ksmbd_vfs_getattr(&fp->filp->f_path, &stat); + if (rc) + goto err_out2; + + goto reconnected_fp; + } + if (dh_info.reconnected == true) { rc = smb2_check_durable_oplock(conn, share, dh_info.fp, lc, sess->user, name); @@ -3601,7 +4037,8 @@ int smb2_open(struct ksmbd_work *work) if (file_present && !(req->CreateOptions & FILE_DELETE_ON_CLOSE_LE)) { rc = smb_check_perm_dacl(conn, &path, &daccess, - sess->user->uid); + req->DesiredAccess, + sess->user->uid, false); if (rc) goto err_out; } @@ -3731,7 +4168,23 @@ int smb2_open(struct ksmbd_work *work) goto err_out; } + /* + * Publish the client and create GUID before an oplock/lease break can + * make this CREATE pending. A replay of that in-flight CREATE must find + * this FP_NEW handle and fail with STATUS_FILE_NOT_AVAILABLE instead of + * waiting on the same break again. + */ + memcpy(fp->client_guid, conn->ClientGUID, SMB2_CLIENT_GUID_SIZE); + if (dh_info.CreateGuid) { + memcpy(fp->create_guid, dh_info.CreateGuid, SMB2_CREATE_GUID_SIZE); + fp->durable_replay_consumed = dh_info.replay_consumed; + rc = ksmbd_vfs_set_durable_owner(fp, sess->user); + if (rc) + goto err_out; + } + fp->cdoption = req->CreateDisposition; + fp->create_file_attributes = req->FileAttributes; fp->daccess = daccess; fp->saccess = req->ShareAccess; fp->coption = req->CreateOptions; @@ -3899,7 +4352,8 @@ int smb2_open(struct ksmbd_work *work) rc = smb_grant_oplock(work, req_op_level, fp->persistent_id, fp, le32_to_cpu(req->hdr.Id.SyncId.TreeId), - lc, share_ret); + lc, share_ret, + smb3_hdr_replay(&req->hdr)); if (rc < 0) goto err_out1; } @@ -3954,7 +4408,32 @@ int smb2_open(struct ksmbd_work *work) query_disk_id = 1; } - if (conn->is_aapl == false) { + if (test_share_config_flag(share, KSMBD_SHARE_FLAG_TIME_MACHINE)) { + context = smb2_find_context_vals(req, SMB2_CREATE_AAPL, 4); + if (IS_ERR(context)) { + rc = PTR_ERR(context); + goto err_out1; + } else if (context) { + struct aapl_server_query_req *aapl_req; + + if (le32_to_cpu(context->DataLength) < + sizeof(struct aapl_server_query_req)) { + rc = -EINVAL; + goto err_out1; + } + + aapl_req = (struct aapl_server_query_req *) + ((char *)context + + le16_to_cpu(context->DataOffset)); + if (le32_to_cpu(aapl_req->cmd) == + SMB2_CRTCTX_AAPL_SERVER_QUERY) { + conn->is_aapl = true; + aapl_ctxt = true; + aapl_req_bitmap = le64_to_cpu(aapl_req->req_bitmap); + aapl_client_caps = le64_to_cpu(aapl_req->client_caps); + } + } + } else if (conn->is_aapl == false) { context = smb2_find_context_vals(req, SMB2_CREATE_AAPL, 4); if (IS_ERR(context)) { rc = PTR_ERR(context); @@ -3987,7 +4466,7 @@ int smb2_open(struct ksmbd_work *work) if (created) smb2_new_xattrs(tcon, &path, fp); - memcpy(fp->client_guid, conn->ClientGUID, SMB2_CLIENT_GUID_SIZE); + fp->create_action = cpu_to_le32(file_info); if (dh_info.type == DURABLE_REQ_V2 || dh_info.type == DURABLE_REQ) { if (dh_info.type == DURABLE_REQ_V2 && dh_info.persistent && @@ -3996,10 +4475,7 @@ int smb2_open(struct ksmbd_work *work) fp->is_persistent = true; else fp->is_durable = true; - if (dh_info.type == DURABLE_REQ_V2) { - memcpy(fp->create_guid, dh_info.CreateGuid, - SMB2_CREATE_GUID_SIZE); if (dh_info.app_instance_id) memcpy(fp->app_instance_id, dh_info.AppInstanceId, @@ -4009,14 +4485,33 @@ int smb2_open(struct ksmbd_work *work) min_t(unsigned int, dh_info.timeout, DURABLE_HANDLE_MAX_TIMEOUT); else - fp->durable_timeout = 60; + fp->durable_timeout = 60000; } } -reconnected_fp: + /* + * conn->is_aapl detection above (this function's create-context + * parsing) is skipped on the reconnect path below, since a + * reconnect always arrives on a fresh connection -- if the client + * cares, it sends its own AAPL context on this same CREATE, which + * this function's normal (non-reconnect) parsing already handles. + */ + reconnected_fp: + if (dh_info.replay) + file_info = le32_to_cpu(fp->create_action); rsp->StructureSize = cpu_to_le16(89); opinfo = opinfo_get(fp); rsp->OplockLevel = opinfo != NULL ? opinfo->level : 0; + /* + * A durable CREATE replay does not modify the existing open. When + * replayed without an oplock, however, its response reflects that + * request and cannot include a new durable-handle response context. + */ + if (dh_info.replay && !lc && + req_op_level == SMB2_OPLOCK_LEVEL_NONE) { + rsp->OplockLevel = SMB2_OPLOCK_LEVEL_NONE; + durable_rsp = false; + } rsp->Flags = 0; rsp->CreateAction = cpu_to_le32(file_info); rsp->CreationTime = cpu_to_le64(fp->create_time); @@ -4110,7 +4605,8 @@ reconnected_fp: next_off = conn->vals->create_disk_id_size; } - if (dh_info.type == DURABLE_REQ || dh_info.type == DURABLE_REQ_V2) { + if (durable_rsp && + (dh_info.type == DURABLE_REQ || dh_info.type == DURABLE_REQ_V2)) { struct create_context *durable_ccontext; durable_ccontext = (struct create_context *)(rsp->Buffer + @@ -4138,6 +4634,10 @@ reconnected_fp: } if (posix_ctxt) { + struct create_context *posix_ccontext; + + posix_ccontext = (struct create_context *)(rsp->Buffer + + le32_to_cpu(rsp->CreateContextsLength)); contxt_cnt++; create_posix_rsp_buf(rsp->Buffer + le32_to_cpu(rsp->CreateContextsLength), @@ -4147,6 +4647,29 @@ reconnected_fp: iov_len += conn->vals->create_posix_size; if (next_ptr) *next_ptr = cpu_to_le32(next_off); + next_ptr = &posix_ccontext->Next; + next_off = conn->vals->create_posix_size; + } + + /* + * AAPL create context response: see smb2pdu.h for the capability + * rationale. Scoped to TIME_MACHINE shares only. + */ + if (aapl_ctxt) { + if (aapl_client_caps & SMB2_CRTCTX_AAPL_SUPPORTS_READ_DIR_ATTR) + conn->aapl_readdir_attr = true; + + contxt_cnt++; + create_aapl_rsp_buf(rsp->Buffer + + le32_to_cpu(rsp->CreateContextsLength), + SMB2_CRTCTX_AAPL_FULL_SYNC, + aapl_req_bitmap); + le32_add_cpu(&rsp->CreateContextsLength, + conn->vals->create_aapl_size); + iov_len += conn->vals->create_aapl_size; + if (next_ptr) + *next_ptr = cpu_to_le32(next_off); + /* AAPL is last; next_ptr need not be updated */ } if (contxt_cnt > 0) { @@ -4163,8 +4686,11 @@ err_out1: err_out2: if (!rc) { - rc = ksmbd_update_fstate(&work->sess->file_table, fp, - FP_INITED); + if (!dh_info.replay) + rc = ksmbd_update_fstate(&work->sess->file_table, fp, + FP_INITED); + if (!rc) + rc = smb2_set_request_open(work, fp, &req->hdr, false, false); if (!rc) rc = ksmbd_iov_pin_rsp(work, (void *)rsp, iov_len); } @@ -4173,8 +4699,12 @@ err_out2: rsp->hdr.Status = STATUS_INVALID_PARAMETER; else if (rc == -EOPNOTSUPP) rsp->hdr.Status = STATUS_NOT_SUPPORTED; - else if (rc == -EACCES || rc == -ESTALE || rc == -EXDEV) - rsp->hdr.Status = STATUS_ACCESS_DENIED; + else if (rc == -EACCES || rc == -ESTALE || rc == -EXDEV) { + if (req->DesiredAccess & FILE_ACCESS_SYSTEM_SECURITY_LE) + rsp->hdr.Status = STATUS_PRIVILEGE_NOT_HELD; + else + rsp->hdr.Status = STATUS_ACCESS_DENIED; + } else if (rc == -ENOENT) rsp->hdr.Status = STATUS_OBJECT_NAME_INVALID; else if (rc == -EPERM) @@ -4191,15 +4721,22 @@ err_out2: rsp->hdr.Status = STATUS_OBJECT_NAME_COLLISION; else if (rc == -EMFILE) rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + else if (rc == -EINPROGRESS) + rsp->hdr.Status = STATUS_FILE_NOT_AVAILABLE; + else if (rc == -EAGAIN) + rsp->hdr.Status = STATUS_FILE_NOT_AVAILABLE; if (!rsp->hdr.Status) rsp->hdr.Status = STATUS_UNEXPECTED_IO_ERROR; - if (fp) + if (fp && !dh_info.replay) ksmbd_fd_put(work, fp); smb2_set_err_rsp(work); ksmbd_debug(SMB, "Error response: %x\n", rsp->hdr.Status); } + if (dh_info.replay) + ksmbd_put_durable_fd(dh_info.fp); + if (dh_info.reconnected) { /* * If reconnect succeeded, fp was republished in the @@ -4572,6 +5109,7 @@ static int process_query_dir_entries(struct smb2_query_dir_private *priv) for (i = 0; i < priv->d_info->num_entry; i++) { struct dentry *dent; + struct path path; if (dentry_name(priv->d_info, priv->info_level)) return -EINVAL; @@ -4594,6 +5132,23 @@ static int process_query_dir_entries(struct smb2_query_dir_private *priv) continue; } + if (test_share_config_flag(priv->work->tcon->share_conf, + KSMBD_SHARE_FLAG_HIDE_UNREADABLE)) { + __le32 daccess = FILE_READ_DATA_LE | FILE_READ_EA_LE | + FILE_READ_ATTRIBUTES_LE; + + path.mnt = priv->dir_fp->filp->f_path.mnt; + path.dentry = dent; + rc = smb_check_perm_dacl(priv->work->conn, &path, + &daccess, daccess, + priv->work->sess->user->uid, + true); + if (rc) { + dput(dent); + continue; + } + } + ksmbd_kstat.kstat = &kstat; if (priv->info_level != FILE_NAMES_INFORMATION) { rc = ksmbd_vfs_fill_dentry_attrs(priv->work, @@ -6405,7 +6960,6 @@ int smb2_close(struct ksmbd_work *work) u64 sess_id; struct smb2_close_req *req; struct smb2_close_rsp *rsp; - struct ksmbd_conn *conn = work->conn; struct ksmbd_file *fp; u64 time; int err = 0; @@ -6428,7 +6982,7 @@ int smb2_close(struct ksmbd_work *work) sess_id = work->compound_sid; work->compound_sid = 0; - if (check_session_id(conn, sess_id)) { + if (work->sess && work->sess->id == sess_id) { work->compound_sid = sess_id; } else { rsp->hdr.Status = STATUS_USER_SESSION_DELETED; @@ -7101,6 +7655,7 @@ int smb2_set_info(struct ksmbd_work *work) struct smb2_set_info_rsp *rsp; struct ksmbd_file *fp = NULL; int rc = 0; + bool chseq_err = false; unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; ksmbd_debug(SMB, "Received smb2 set info request\n"); @@ -7140,6 +7695,13 @@ int smb2_set_info(struct ksmbd_work *work) goto err_out; } + rc = smb2_set_request_open(work, fp, &req->hdr, true, false); + if (rc) { + rsp->hdr.Status = STATUS_FILE_NOT_AVAILABLE; + chseq_err = true; + goto err_out; + } + saved_cred = override_creds(fp->filp->f_cred); switch (req->InfoType) { case SMB2_O_INFO_FILE: @@ -7184,7 +7746,7 @@ err_out: rsp->hdr.Status = STATUS_OBJECT_NAME_INVALID; else if (rc == -EBUSY || rc == -ENOTEMPTY) rsp->hdr.Status = STATUS_DIRECTORY_NOT_EMPTY; - else if (rc == -EAGAIN) + else if (rc == -EAGAIN && !chseq_err) rsp->hdr.Status = STATUS_FILE_LOCK_CONFLICT; else if (rc == -EBADF || rc == -ESTALE) rsp->hdr.Status = STATUS_INVALID_HANDLE; @@ -7398,6 +7960,10 @@ int smb2_read(struct ksmbd_work *work) goto out; } + err = smb2_set_request_open(work, fp, &req->hdr, true, true); + if (err) + goto out; + if (!(fp->daccess & (FILE_READ_DATA_LE | FILE_READ_ATTRIBUTES_LE))) { pr_err("Not permitted to read : 0x%x\n", fp->daccess); err = -EACCES; @@ -7645,6 +8211,7 @@ int smb2_write(struct ksmbd_work *work) char *data_buf; bool writethrough = false, is_rdma_channel = false; bool async_interim = false; + bool chseq_err = false; int err = 0; unsigned int max_write_size = work->conn->vals->max_write_size; unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; @@ -7675,8 +8242,10 @@ int smb2_write(struct ksmbd_work *work) } offset = le64_to_cpu(req->Offset); - if (offset < 0) - return -EINVAL; + if (offset < 0) { + err = -EINVAL; + goto out; + } length = le32_to_cpu(req->Length); if (req->Channel == SMB2_CHANNEL_RDMA_V1 || @@ -7690,6 +8259,19 @@ int smb2_write(struct ksmbd_work *work) length = le32_to_cpu(req->RemainingBytes); } + if (length) { + u64 end = (u64)offset + length; + + if (end > SMB2_MAX_FILE_SIZE) { + err = -EINVAL; + goto out; + } + if (end == SMB2_MAX_FILE_SIZE) { + err = -EFBIG; + goto out; + } + } + if (is_rdma_channel == true) { unsigned int ch_offset = le16_to_cpu(req->WriteChannelInfoOffset); @@ -7719,6 +8301,13 @@ int smb2_write(struct ksmbd_work *work) goto out; } + err = smb2_set_request_open(work, fp, &req->hdr, true, false); + if (err) { + rsp->hdr.Status = STATUS_FILE_NOT_AVAILABLE; + chseq_err = true; + goto out; + } + if (!(fp->daccess & (FILE_WRITE_DATA_LE | FILE_READ_ATTRIBUTES_LE))) { pr_err("Not permitted to write : 0x%x\n", fp->daccess); err = -EACCES; @@ -7790,7 +8379,7 @@ out: if (async_interim) release_async_work(work); - if (err == -EAGAIN) + if (err == -EAGAIN && !chseq_err) rsp->hdr.Status = STATUS_FILE_LOCK_CONFLICT; else if (err == -ENOSPC || err == -EFBIG) rsp->hdr.Status = STATUS_DISK_FULL; @@ -7802,7 +8391,7 @@ out: rsp->hdr.Status = STATUS_SHARING_VIOLATION; else if (err == -EINVAL) rsp->hdr.Status = STATUS_INVALID_PARAMETER; - else + else if (rsp->hdr.Status == 0) rsp->hdr.Status = STATUS_INVALID_HANDLE; smb2_set_err_rsp(work); @@ -7869,6 +8458,7 @@ int smb2_cancel(struct ksmbd_work *work) struct smb2_hdr *hdr = smb_get_msg(work->request_buf); struct smb2_hdr *chdr; struct ksmbd_work *iter; + struct ksmbd_work *cancelled_notify = NULL; struct list_head *command_list; if (work->next_smb2_rcv_hdr_off) @@ -7906,11 +8496,23 @@ int smb2_cancel(struct ksmbd_work *work) le64_to_cpu(hdr->Id.AsyncId), le16_to_cpu(chdr->Command)); iter->state = KSMBD_WORK_CANCELLED; - if (iter->cancel_fn) + if (iter->cancel_fn == smb2_notify_cancel_fn) + cancelled_notify = + smb2_notify_cancel_claim(iter->cancel_argv); + else if (iter->cancel_fn) iter->cancel_fn(iter->cancel_argv); break; } spin_unlock(&conn->request_lock); + + /* + * Complete a cancelled notify before this CANCEL handler returns. + * Deferring it to the system workqueue lets a following request and + * its response overtake STATUS_CANCELLED, leaving clients waiting + * for the original notify even though the cancellation was accepted. + */ + if (cancelled_notify) + smb2_complete_notify_cancel(cancelled_notify); } else { command_list = &conn->requests; @@ -8031,6 +8633,19 @@ static void smb2_remove_blocked_lock(void **argv) locks_wake_up(flock); } +static void smb2_free_lock(struct file_lock *flock) +{ + ksmbd_vfs_posix_lock_unblock(flock); + locks_free_lock(flock); +} + +static void smb2_free_blocked_lock(struct file_lock *flock) +{ + ksmbd_vfs_posix_lock_unblock(flock); + locks_wake_up(flock); + locks_free_lock(flock); +} + static inline bool lock_defer_pending(struct file_lock *fl) { /* check pending lock waiters */ @@ -8063,6 +8678,7 @@ int smb2_lock(struct ksmbd_work *work) LIST_HEAD(rollback_list); int prior_lock = 0, bkt; unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; + bool lock_replayed; WORK_BUFFERS(work, req, rsp); @@ -8091,6 +8707,14 @@ int smb2_lock(struct ksmbd_work *work) goto out2; } + err = smb2_set_request_open(work, fp, &req->hdr, false, false); + if (err) + goto out2; + + lock_replayed = smb2_verify_lock_sequence(work, fp, req); + if (lock_replayed) + goto lock_success; + filp = fp->filp; lock_count = le16_to_cpu(req->LockCount); lock_ele = req->locks; @@ -8207,14 +8831,14 @@ int smb2_lock(struct ksmbd_work *work) cmp_lock->end == smb_lock->end && !lock_defer_pending(cmp_lock->fl)) { nolock = 0; - list_del(&cmp_lock->flist); - list_del(&cmp_lock->clist); + list_del_init(&cmp_lock->flist); + list_del_init(&cmp_lock->clist); cmp_lock->conn = NULL; spin_unlock(&conn->llist_lock); up_read(&conn_list_lock); ksmbd_conn_put(conn); - locks_free_lock(cmp_lock->fl); + smb2_free_lock(cmp_lock->fl); kfree(cmp_lock); goto out_check_cl; } @@ -8296,11 +8920,12 @@ skip: ksmbd_debug(SMB, "would have to wait for getting lock\n"); - list_add(&smb_lock->llist, &rollback_list); argv = kmalloc(sizeof(void *), KSMBD_DEFAULT_GFP); if (!argv) { err = -ENOMEM; + smb2_free_blocked_lock(flock); + kfree(smb_lock); goto out; } argv[0] = flock; @@ -8311,8 +8936,11 @@ skip: if (rc) { kfree(argv); err = -ENOMEM; + smb2_free_blocked_lock(flock); + kfree(smb_lock); goto out; } + list_add(&smb_lock->llist, &rollback_list); spin_lock(&fp->f_lock); list_add(&work->fp_entry, &fp->blocked_works); spin_unlock(&fp->f_lock); @@ -8345,16 +8973,10 @@ skip: rsp->hdr.Status = STATUS_RANGE_NOT_LOCKED; kfree(smb_lock); - goto out2; + /* rollback_list may still hold earlier grants */ + goto out; } else if (!rc) { list_add(&smb_lock->llist, &rollback_list); - smb_lock->conn = ksmbd_conn_get(work->conn); - spin_lock(&work->conn->llist_lock); - list_add_tail(&smb_lock->clist, - &work->conn->lock_list); - list_add_tail(&smb_lock->flist, - &fp->lock_list); - spin_unlock(&work->conn->llist_lock); ksmbd_debug(SMB, "successful in taking lock\n"); } else { locks_free_lock(flock); @@ -8368,6 +8990,7 @@ skip: if (atomic_read(&fp->f_ci->op_count) > 1) smb_break_all_oplock(work, fp); +lock_success: rsp->StructureSize = cpu_to_le16(4); ksmbd_debug(SMB, "successful in taking lock\n"); rsp->hdr.Status = STATUS_SUCCESS; @@ -8376,6 +8999,23 @@ skip: if (err) goto out; + /* publish only once the whole batch has committed */ + if (!list_empty(&rollback_list)) { + spin_lock(&work->conn->llist_lock); + list_for_each_entry_safe(smb_lock, tmp, &rollback_list, llist) { + list_del_init(&smb_lock->llist); + smb_lock->conn = ksmbd_conn_get(work->conn); + list_add_tail(&smb_lock->clist, + &work->conn->lock_list); + list_add_tail(&smb_lock->flist, + &fp->lock_list); + } + spin_unlock(&work->conn->llist_lock); + } + + if (!lock_replayed) + smb2_update_lock_sequence(work, fp, req); + ksmbd_fd_put(work, fp); return 0; @@ -8403,16 +9043,7 @@ out: } list_del(&smb_lock->llist); - conn = smb_lock->conn; - spin_lock(&conn->llist_lock); - if (!list_empty(&smb_lock->flist)) - list_del(&smb_lock->flist); - list_del(&smb_lock->clist); - smb_lock->conn = NULL; - spin_unlock(&conn->llist_lock); - ksmbd_conn_put(conn); - - locks_free_lock(smb_lock->fl); + smb2_free_lock(smb_lock->fl); if (rlock) locks_free_lock(rlock); kfree(smb_lock); @@ -8880,10 +9511,12 @@ int smb2_ioctl(struct ksmbd_work *work) struct smb2_ioctl_req *req; struct smb2_ioctl_rsp *rsp; unsigned int cnt_code, nbytes = 0, out_buf_len, in_buf_len; - u64 id = KSMBD_NO_FID; + u64 id = KSMBD_NO_FID, pid = KSMBD_NO_FID; struct ksmbd_conn *conn = work->conn; int ret = 0; char *buffer; + bool no_fileid_ioctl = false; + bool chseq_err = false; ksmbd_debug(SMB, "Received smb2 ioctl request\n"); @@ -8896,14 +9529,17 @@ int smb2_ioctl(struct ksmbd_work *work) ksmbd_debug(SMB, "Compound request set FID = %llu\n", work->compound_fid); id = work->compound_fid; + pid = work->compound_pfid; } } else { req = smb_get_msg(work->request_buf); rsp = smb_get_msg(work->response_buf); } - if (!has_file_id(id)) + if (!has_file_id(id)) { id = req->VolatileFileId; + pid = req->PersistentFileId; + } if (req->Flags != cpu_to_le32(SMB2_0_IOCTL_IS_FSCTL)) { ret = -EOPNOTSUPP; @@ -8913,6 +9549,40 @@ int smb2_ioctl(struct ksmbd_work *work) buffer = (char *)req + le32_to_cpu(req->InputOffset); cnt_code = le32_to_cpu(req->CtlCode); + switch (cnt_code) { + case FSCTL_DFS_GET_REFERRALS: + case FSCTL_DFS_GET_REFERRALS_EX: + case FSCTL_QUERY_NETWORK_INTERFACE_INFO: + case FSCTL_VALIDATE_NEGOTIATE_INFO: + case FSCTL_PIPE_WAIT: + no_fileid_ioctl = true; + break; + default: + break; + } + + if (!no_fileid_ioctl && has_file_id(id)) { + struct ksmbd_file *fp; + + fp = ksmbd_lookup_fd_slow(work, id, pid); + if (!fp) { + if (cnt_code == FSCTL_DUPLICATE_EXTENTS_TO_FILE) { + rsp->hdr.Status = STATUS_FILE_CLOSED; + goto out2; + } + ret = -ENOENT; + goto out; + } + + ret = smb2_set_request_open(work, fp, &req->hdr, true, false); + ksmbd_fd_put(work, fp); + if (ret) { + rsp->hdr.Status = STATUS_FILE_NOT_AVAILABLE; + chseq_err = true; + goto out; + } + } + ret = smb2_calc_max_out_buf_len(work, offsetof(struct smb2_ioctl_rsp, Buffer), le32_to_cpu(req->MaxOutputResponse)); @@ -9303,7 +9973,7 @@ out: rsp->hdr.Status = STATUS_NOT_SUPPORTED; else if (ret == -ENOSPC) rsp->hdr.Status = STATUS_BUFFER_TOO_SMALL; - else if (ret < 0 || rsp->hdr.Status == 0) + else if (!chseq_err && (ret < 0 || rsp->hdr.Status == 0)) rsp->hdr.Status = STATUS_INVALID_PARAMETER; out2: @@ -9343,6 +10013,14 @@ static void smb20_oplock_break_ack(struct ksmbd_work *work) return; } + ret = smb2_set_request_open(work, fp, &req->hdr, false, false); + if (ret) { + rsp->hdr.Status = STATUS_FILE_CLOSED; + smb2_set_err_rsp(work); + ksmbd_fd_put(work, fp); + return; + } + opinfo = opinfo_get(fp); if (!opinfo) { pr_err("unexpected null oplock_info\n"); @@ -9355,6 +10033,22 @@ static void smb20_oplock_break_ack(struct ksmbd_work *work) if (opinfo->op_state != OPLOCK_ACK_WAIT) { ksmbd_debug(SMB, "unexpected oplock state 0x%x\n", opinfo->op_state); + if (smb3_hdr_replay(&req->hdr) && + opinfo->op_state == OPLOCK_STATE_NONE) { + rsp->StructureSize = cpu_to_le16(24); + rsp->OplockLevel = opinfo->level; + rsp->Reserved = 0; + rsp->Reserved2 = 0; + rsp->VolatileFid = volatile_id; + rsp->PersistentFid = persistent_id; + ret = ksmbd_iov_pin_rsp(work, rsp, + sizeof(struct smb2_oplock_break)); + if (ret) + ksmbd_debug(SMB, + "failed to pin replayed oplock break response: %d\n", + ret); + goto out_no_state_change; + } if (opinfo->level == SMB2_OPLOCK_LEVEL_NONE) status = STATUS_INVALID_OPLOCK_PROTOCOL; else @@ -9421,8 +10115,12 @@ err_out: smb2_set_err_rsp(work); out: - opinfo->op_state = OPLOCK_STATE_NONE; + spin_lock(&opinfo->state_lock); + if (opinfo->op_state != OPLOCK_CLOSING) + opinfo->op_state = OPLOCK_STATE_NONE; + spin_unlock(&opinfo->state_lock); wake_up_interruptible_all(&opinfo->oplock_q); +out_no_state_change: opinfo_put(opinfo); ksmbd_fd_put(work, fp); } @@ -9475,11 +10173,15 @@ static void smb21_lease_break_ack(struct ksmbd_work *work) if (opinfo->op_state == OPLOCK_STATE_NONE) { pr_err("unexpected lease break state 0x%x\n", opinfo->op_state); + if (smb3_hdr_replay(&req->hdr)) + goto replay_rsp; rsp->hdr.Status = STATUS_UNSUCCESSFUL; goto err_out; } if (!atomic_read(&opinfo->breaking_cnt)) { + if (smb3_hdr_replay(&req->hdr)) + goto replay_rsp; rsp->hdr.Status = STATUS_UNSUCCESSFUL; goto err_out; } @@ -9507,13 +10209,29 @@ static void smb21_lease_break_ack(struct ksmbd_work *work) if (ret) goto err_out; - opinfo->op_state = OPLOCK_STATE_NONE; + spin_lock(&opinfo->state_lock); + if (opinfo->op_state != OPLOCK_CLOSING) + opinfo->op_state = OPLOCK_STATE_NONE; + spin_unlock(&opinfo->state_lock); wake_up_interruptible_all(&opinfo->oplock_q); - atomic_dec(&opinfo->breaking_cnt); + atomic_dec_if_positive(&opinfo->breaking_cnt); wake_up_interruptible_all(&opinfo->oplock_brk); opinfo_put(opinfo); return; +replay_rsp: + rsp->StructureSize = cpu_to_le16(36); + rsp->Reserved = 0; + rsp->Flags = 0; + memcpy(rsp->LeaseKey, req->LeaseKey, 16); + rsp->LeaseState = lease->state; + rsp->LeaseDuration = 0; + ret = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_lease_ack)); + if (ret) + goto err_out; + opinfo_put(opinfo); + return; + err_out: smb2_set_err_rsp(work); opinfo_put(opinfo); @@ -9553,6 +10271,127 @@ int smb2_oplock_break(struct ksmbd_work *work) return 0; } +/* + * Cancel handler for a deferred CHANGE_NOTIFY. Races against + * __ksmbd_close_fd()'s notify_pendings drain (vfs_cache.c), which can run + * concurrently on a different connection closing the same handle -- only + * one of the two may claim and free in_work, so both sides check + * list_empty() under fp->f_lock before touching it (list_del_init() + * leaves a node empty, so whichever side removes it first is the owner; + * the loser must not touch in_work again, since the winner may already be + * freeing it). + * + * smb2_cancel() holds conn->request_lock (a spinlock) for the entire + * time it walks conn->async_requests and calls this function -- so this + * runs with preemption disabled and must not sleep or re-acquire that + * same lock. release_async_work() does both (it takes conn->request_lock + * itself, and frees things that can involve sleeping paths), so calling + * it from here would self-deadlock the very thread processing the + * client's CANCEL command. ksmbd_conn_write() can also sleep (it takes + * conn's write mutex). So: do only the non-sleeping, no-relock cleanup + * inline here. smb2_cancel() sends and frees the claimed notify after it + * drops request_lock, preserving response order for a client CANCEL. The + * connection teardown caller has no such post-unlock path, so its wrapper + * defers the send and free to a workqueue. + */ +struct notify_cancel_ctx { + struct work_struct work; + struct ksmbd_work *in_work; +}; + +static void smb2_send_notify_cancelled(struct ksmbd_work *work) +{ + struct smb2_hdr *hdr = smb_get_msg(work->response_buf); + struct ksmbd_conn *conn = work->conn; + struct ksmbd_session *sess; + + sess = ksmbd_session_lookup(conn, le64_to_cpu(hdr->SessionId)); + if (sess) { + work->sess = sess; + if (work->encrypted && sess->enc && conn->ops->encrypt_resp) { + conn->ops->encrypt_resp(work); + } else if (conn->ops->is_sign_req && conn->ops->set_sign_rsp && + conn->ops->is_sign_req(work, + conn->ops->get_cmd_val(work))) { + conn->ops->set_sign_rsp(work); + } + } + + ksmbd_conn_write(work); + if (sess) { + ksmbd_user_session_put(sess); + work->sess = NULL; + } +} + +static void smb2_notify_cancel_deferred(struct work_struct *w) +{ + struct notify_cancel_ctx *ctx = + container_of(w, struct notify_cancel_ctx, work); + + smb2_complete_notify_cancel(ctx->in_work); + kfree(ctx); +} + +static struct ksmbd_work *smb2_notify_cancel_claim(void **argv) +{ + struct ksmbd_work *in_work = (struct ksmbd_work *)argv[0]; + struct ksmbd_file *fp = (struct ksmbd_file *)argv[1]; + bool claimed; + + spin_lock(&fp->f_lock); + claimed = !list_empty(&in_work->notify_entry); + if (claimed) + list_del_init(&in_work->notify_entry); + spin_unlock(&fp->f_lock); + + if (!claimed) + return NULL; + + /* conn->request_lock is held by smb2_cancel() or connection teardown. */ + in_work->cancel_fn = NULL; + kfree(in_work->cancel_argv); + in_work->cancel_argv = NULL; + return in_work; +} + +static void smb2_complete_notify_cancel(struct ksmbd_work *in_work) +{ + struct smb2_hdr *in_hdr = smb_get_msg(in_work->response_buf); + + in_hdr->Status = STATUS_CANCELLED; + smb2_send_notify_cancelled(in_work); + release_async_work(in_work); + ksmbd_free_work_struct(in_work); +} + +static void smb2_notify_cancel_fn(void **argv) +{ + struct ksmbd_work *in_work = smb2_notify_cancel_claim(argv); + struct ksmbd_conn *conn; + struct notify_cancel_ctx *ctx; + + if (!in_work) + return; + conn = in_work->conn; + + ctx = kmalloc(sizeof(*ctx), GFP_ATOMIC); + if (!ctx) { + /* Can't defer the response -- free without sending one. */ + list_del_init(&in_work->async_request_entry); + in_work->asynchronous = false; + if (in_work->async_id) { + ksmbd_release_id(&conn->async_ida, in_work->async_id); + in_work->async_id = 0; + } + ksmbd_free_work_struct(in_work); + return; + } + ctx->in_work = in_work; + INIT_WORK(&ctx->work, smb2_notify_cancel_deferred); + schedule_work(&ctx->work); +} + /** * smb2_notify() - handler for smb2 notify request * @work: smb work containing notify command buffer @@ -9563,6 +10402,9 @@ int smb2_notify(struct ksmbd_work *work) { struct smb2_change_notify_req *req; struct smb2_change_notify_rsp *rsp; + struct ksmbd_work *in_work; + struct smb2_hdr *in_hdr; + struct ksmbd_file *fp; ksmbd_debug(SMB, "Received smb2 notify\n"); @@ -9577,9 +10419,149 @@ int smb2_notify(struct ksmbd_work *work) return -EIO; } - smb2_set_err_rsp(work); - rsp->hdr.Status = STATUS_NOT_IMPLEMENTED; - return -EOPNOTSUPP; + /* + * macOS backupd sends CHANGE_NOTIFY with FileId=FFFF...FFFF (share-root + * sentinel) to watch for changes on the share root without holding an + * open handle. Respond STATUS_PENDING + STATUS_NOTIFY_CLEANUP immediately; + * without this, backupd aborts Time Machine setup on STATUS_FILE_CLOSED. + */ + if (req->VolatileFileId == SMB2_NO_FID && + req->PersistentFileId == SMB2_NO_FID) { + in_work = ksmbd_alloc_work_struct(); + if (!in_work || allocate_interim_rsp_buf(in_work)) { + if (in_work) + ksmbd_free_work_struct(in_work); + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + smb2_set_err_rsp(work); + return 0; + } + if (setup_async_work(work, NULL, NULL)) { + ksmbd_free_work_struct(in_work); + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + smb2_set_err_rsp(work); + return 0; + } + smb2_send_interim_resp(work, STATUS_PENDING); + in_work->conn = work->conn; + in_hdr = smb_get_msg(in_work->response_buf); + memcpy(in_hdr, ksmbd_resp_buf_next(work), + __SMB2_HEADER_STRUCTURE_SIZE); + in_hdr->Flags |= SMB2_FLAGS_ASYNC_COMMAND; + in_hdr->Id.AsyncId = cpu_to_le64(work->async_id); + smb2_set_err_rsp(in_work); + in_hdr->Status = STATUS_NOTIFY_CLEANUP; + in_work->async_id = work->async_id; + work->async_id = 0; + release_async_work(work); + ksmbd_conn_write(in_work); + ksmbd_free_work_struct(in_work); + work->send_no_response = 1; + return 0; + } + + /* + * KSMBD does not implement a real change-notification backend. + * Genuine SMB2 servers (and macOS smbfs) never complete a + * CHANGE_NOTIFY spontaneously: it is satisfied only by a real + * directory change, or with STATUS_NOTIFY_CLEANUP when the watched + * handle is closed. Completing it early (e.g. on a timer) makes + * Finder treat the cleanup as "directory changed" and re-enumerate + * the directory forever, leaving items unopenable. Returning + * STATUS_NOT_IMPLEMENTED here (like stock ksmbd) makes macOS smbfs + * hard-freeze on unmount, so this must stay deferred. + */ + fp = ksmbd_lookup_fd_slow(work, req->VolatileFileId, req->PersistentFileId); + if (!fp) { + rsp->hdr.Status = STATUS_FILE_CLOSED; + smb2_set_err_rsp(work); + return 0; + } + + in_work = ksmbd_alloc_work_struct(); + if (!in_work || allocate_interim_rsp_buf(in_work)) { + if (in_work) + ksmbd_free_work_struct(in_work); + ksmbd_fd_put(work, fp); + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + smb2_set_err_rsp(work); + return 0; + } + /* + * in_work is synthetic (not from the normal request-receiving + * pipeline), so it has no request_buf of its own. It gets registered + * into conn->async_requests below, and smb2_cancel() unconditionally + * computes smb_get_msg(iter->request_buf) for every entry in that + * list while searching for a match -- give it its own small buffer + * (not an alias of response_buf: ksmbd_free_work_struct() kvfree()s + * both separately, so aliasing them would double-free) so that stays + * a harmless read instead of a near-NULL dereference. + */ + in_work->request_buf = kzalloc(MAX_CIFS_SMALL_BUFFER_SIZE, KSMBD_DEFAULT_GFP); + if (!in_work->request_buf) { + ksmbd_free_work_struct(in_work); + ksmbd_fd_put(work, fp); + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + smb2_set_err_rsp(work); + return 0; + } + memcpy(smb_get_msg(in_work->request_buf), req, + __SMB2_HEADER_STRUCTURE_SIZE); + + if (setup_async_work(work, NULL, NULL)) { + ksmbd_free_work_struct(in_work); + ksmbd_fd_put(work, fp); + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + smb2_set_err_rsp(work); + return 0; + } + + smb2_send_interim_resp(work, STATUS_PENDING); + + /* Keep the async IDA alive until the deferred work is released. */ + in_work->conn = ksmbd_conn_get(work->conn); + in_work->owns_conn_ref = true; + in_work->encrypted = work->encrypted; + in_hdr = smb_get_msg(in_work->response_buf); + memcpy(in_hdr, ksmbd_resp_buf_next(work), __SMB2_HEADER_STRUCTURE_SIZE); + in_hdr->Flags |= SMB2_FLAGS_ASYNC_COMMAND; + in_hdr->Id.AsyncId = cpu_to_le64(work->async_id); + smb2_set_err_rsp(in_work); + in_hdr->Status = STATUS_NOTIFY_CLEANUP; + + /* + * Transfer ownership of the async id to in_work; it stays reserved + * until in_work is freed after the deferred response is sent on + * close, so it can't be reused for an unrelated async response. + */ + in_work->async_id = work->async_id; + work->async_id = 0; + release_async_work(work); + + /* + * work itself is about to be recycled by the normal request-processing + * pipeline, so it can't stay the target of a future CANCEL -- register + * in_work instead, reusing the same async_id, so a client-sent CANCEL + * for this notify actually finds something to cancel instead of + * silently doing nothing until the handle eventually closes. + */ + in_work->asynchronous = true; + in_work->cancel_argv = kmalloc_array(2, sizeof(void *), KSMBD_DEFAULT_GFP); + if (in_work->cancel_argv) { + in_work->cancel_argv[0] = in_work; + in_work->cancel_argv[1] = fp; + in_work->cancel_fn = smb2_notify_cancel_fn; + } + spin_lock(&work->conn->request_lock); + list_add_tail(&in_work->async_request_entry, &work->conn->async_requests); + spin_unlock(&work->conn->request_lock); + + spin_lock(&fp->f_lock); + list_add_tail(&in_work->notify_entry, &fp->notify_pendings); + spin_unlock(&fp->f_lock); + + ksmbd_fd_put(work, fp); + work->send_no_response = 1; + return 0; } /** @@ -9872,33 +10854,33 @@ bool smb3_is_transform_hdr(void *buf) int smb3_decrypt_req(struct ksmbd_work *work) { - struct ksmbd_session *sess; char *buf = work->request_buf; unsigned int pdu_length = get_rfc1002_len(buf); struct kvec iov[2]; - int buf_data_size = pdu_length - sizeof(struct smb2_transform_hdr); + unsigned int buf_data_size; struct smb2_transform_hdr *tr_hdr = smb_get_msg(buf); + unsigned int original_msg_size; int rc = 0; - if (pdu_length < sizeof(struct smb2_transform_hdr) || - buf_data_size < sizeof(struct smb2_hdr)) { + if (pdu_length < sizeof(struct smb2_transform_hdr)) { pr_err("Transform message is too small (%u)\n", pdu_length); return -ECONNABORTED; } - if (buf_data_size < le32_to_cpu(tr_hdr->OriginalMessageSize)) { - pr_err("Transform message is broken\n"); + buf_data_size = pdu_length - sizeof(struct smb2_transform_hdr); + original_msg_size = le32_to_cpu(tr_hdr->OriginalMessageSize); + if (buf_data_size < sizeof(struct smb2_compression_hdr) || + original_msg_size < sizeof(struct smb2_compression_hdr)) { + pr_err("Transform message is too small (%u)\n", + pdu_length); return -ECONNABORTED; } - sess = ksmbd_session_lookup_all(work->conn, le64_to_cpu(tr_hdr->SessionId)); - if (!sess) { - pr_err("invalid session id(%llx) in transform header\n", - le64_to_cpu(tr_hdr->SessionId)); + if (buf_data_size < original_msg_size) { + pr_err("Transform message is broken\n"); return -ECONNABORTED; } - ksmbd_user_session_put(sess); iov[0].iov_base = buf; iov[0].iov_len = sizeof(struct smb2_transform_hdr) + 4; @@ -9908,8 +10890,9 @@ int smb3_decrypt_req(struct ksmbd_work *work) if (rc) return rc; - memmove(buf + 4, iov[1].iov_base, buf_data_size); - *(__be32 *)buf = cpu_to_be32(buf_data_size); + /* Drop the AEAD authentication tag from the inner RFC1002 frame. */ + memmove(buf + 4, iov[1].iov_base, original_msg_size); + *(__be32 *)buf = cpu_to_be32(original_msg_size); return rc; } diff --git a/fs/smb/server/smb2pdu.h b/fs/smb/server/smb2pdu.h index aa06c8c905f1fc..1836259be89713 100644 --- a/fs/smb/server/smb2pdu.h +++ b/fs/smb/server/smb2pdu.h @@ -66,6 +66,65 @@ struct preauth_integrity_info { /* Apple Defined Contexts */ #define SMB2_CREATE_AAPL "AAPL" +/* + * AAPL SMB2 extension -- kAAPL_SERVER_QUERY create context. + * + * Command code and bitmap values are the existing + * SMB2_CRTCTX_AAPL_* constants in fs/smb/common/smb2pdu.h. + * + * Omitting the model string when reply_bitmap includes + * SMB2_CRTCTX_AAPL_MODEL_INFO causes smbfs.kext to enter a broken + * disconnect path requiring a reboot. + * + * Layout: ccontext(16) + Name[4] + Pad[4] + cmd(4) + reserved(4) + + * reply_bitmap(8) + server_caps(8) + vol_caps(8) + * When MODEL_INFO requested, appended: pad2(4) + model_bytes(4) + UTF-16LE + */ +#define SMB2_CREATE_AAPL_LEN 4 + +/* + * Server capability flags (server_caps field) -- SMB2_CRTCTX_AAPL_UNIX_BASED: + * prevents macOS Windows-compat mode (question-mark icons). + * SMB2_CRTCTX_AAPL_SUPPORTS_OSX_COPYFILE: enables server-side file copy via + * FSCTL_SRV_COPYCHUNK. SMB2_CRTCTX_AAPL_SUPPORTS_READ_DIR_ATTR: inline + * FinderInfo per FIND entry, set when client also advertises the bit; + * format: EaSize=max_access, ShortName[0..7]=rfork_size, + * ShortName[8..23]=FinderInfo(16B), Reserved2=unix_mode. + */ +#define AAPL_SERVER_CAPS_KSMBD (SMB2_CRTCTX_AAPL_UNIX_BASED | \ + SMB2_CRTCTX_AAPL_SUPPORTS_OSX_COPYFILE | \ + SMB2_CRTCTX_AAPL_SUPPORTS_READ_DIR_ATTR) + +/* Model string: up to 31 ASCII chars */ +#define AAPL_MODEL_MAX_CHARS 31 +#define AAPL_MODEL_UTF16_BYTES (AAPL_MODEL_MAX_CHARS * 2) + +/* + * Max AAPL response: header(24) + base data(32) + pad2(4) + model_bytes(4) + * + model(62), 8-byte aligned: ALIGN(126, 8) = 128 bytes. + */ +#define AAPL_RSP_MAX_SIZE 128 + +/* AAPL server query request (client->server) */ +struct aapl_server_query_req { + __le32 cmd; + __le32 reserved; + __le64 req_bitmap; + __le64 client_caps; +} __packed; + +struct create_aapl_rsp { + struct create_context_hdr ccontext; + __u8 Name[4]; + __u8 Pad[4]; + __le32 cmd; + __le32 reserved; + __le64 reply_bitmap; + __le64 server_caps; + __le64 vol_caps; + /* when MODEL_INFO requested: __le32 pad2; __le32 model_bytes; __le16 model[] */ +} __packed; + #define DURABLE_HANDLE_MAX_TIMEOUT 300000 struct create_alloc_size_req { @@ -347,6 +406,7 @@ bool smb3_encryption_negotiated(struct ksmbd_conn *conn); /* smb2 misc functions */ int ksmbd_smb2_check_message(struct ksmbd_work *work); +void smb2_complete_request_open(struct ksmbd_work *work); /* smb2 command handlers */ int smb2_handle_negotiate(struct ksmbd_work *work); diff --git a/fs/smb/server/smbacl.c b/fs/smb/server/smbacl.c index c13f07a09ab8b4..33825c1356b477 100644 --- a/fs/smb/server/smbacl.c +++ b/fs/smb/server/smbacl.c @@ -1432,7 +1432,8 @@ bool smb_inherit_flags(int flags, bool is_dir) } int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, - __le32 *pdaccess, int uid) + __le32 *pdaccess, __le32 raw_daccess, int uid, + bool strict) { struct mnt_idmap *idmap = mnt_idmap(path->mnt); struct smb_ntsd *pntsd = NULL; @@ -1442,10 +1443,11 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, unsigned int dacl_offset; size_t dacl_struct_end; struct smb_sid sid; - int granted = le32_to_cpu(*pdaccess & ~FILE_MAXIMAL_ACCESS_LE); + int requested = le32_to_cpu(*pdaccess & ~FILE_MAXIMAL_ACCESS_LE); + int granted = requested; struct smb_ace *ace; int i, found = 0; - unsigned int access_bits = 0; + unsigned int access_bits = 0, denied = 0; struct smb_ace *others_ace = NULL; struct posix_acl_entry *pa_entry; unsigned int sid_type = SIDOWNER; @@ -1479,12 +1481,15 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, goto err_out; } + if (!uid) + sid_type = SIDUNIX_USER; + id_to_sid(uid, sid_type, &sid); + if (*pdaccess & FILE_MAXIMAL_ACCESS_LE) { - granted = READ_CONTROL | WRITE_DAC | FILE_READ_ATTRIBUTES | + access_bits = READ_CONTROL | WRITE_DAC | FILE_READ_ATTRIBUTES | DELETE; - ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl)); - aces_size = acl_size - sizeof(struct smb_acl); + aces_size = pdacl_size - sizeof(struct smb_acl); for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) { if (aces_size < offsetof(struct smb_ace, sid) + CIFS_SID_BASE_SIZE) @@ -1495,17 +1500,43 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, CIFS_SID_BASE_SIZE) break; aces_size -= ace_size; - granted |= le32_to_cpu(ace->access_req); + + if (ace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES || + ace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE + + sizeof(__le32) * ace->sid.num_subauth) + break; + + if (ace->flags & INHERIT_ONLY_ACE || + (compare_sids(&sid, &ace->sid) && + compare_sids(&sid_unix_NFS_mode, &ace->sid) && + compare_sids(&sid_everyone, &ace->sid) && + compare_sids(&sid_authusers, &ace->sid))) + goto next_ace; + + switch (ace->type) { + case ACCESS_ALLOWED_ACE_TYPE: + access_bits |= le32_to_cpu(ace->access_req); + break; + case ACCESS_DENIED_ACE_TYPE: + case ACCESS_DENIED_CALLBACK_ACE_TYPE: + denied |= ~access_bits & + le32_to_cpu(ace->access_req); + break; + } +next_ace: ace = (struct smb_ace *)((char *)ace + le16_to_cpu(ace->size)); } + access_bits &= ~denied; + if ((raw_daccess & FILE_GENERIC_EXECUTE_LE) && + S_ISREG(d_inode(path->dentry)->i_mode) && + (access_bits & GENERIC_READ_FLAGS) == GENERIC_READ_FLAGS) + access_bits |= FILE_EXECUTE; + granted = requested | access_bits; } - if (!uid) - sid_type = SIDUNIX_USER; - id_to_sid(uid, sid_type, &sid); - ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl)); - aces_size = acl_size - sizeof(struct smb_acl); + aces_size = pdacl_size - sizeof(struct smb_acl); for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) { if (aces_size < offsetof(struct smb_ace, sid) + CIFS_SID_BASE_SIZE) @@ -1527,25 +1558,16 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, found = 1; break; } - if (!compare_sids(&sid_everyone, &ace->sid)) + if (!compare_sids(&sid_everyone, &ace->sid) || + !compare_sids(&sid_authusers, &ace->sid)) others_ace = ace; ace = (struct smb_ace *)((char *)ace + le16_to_cpu(ace->size)); } - if (*pdaccess & FILE_MAXIMAL_ACCESS_LE && found) { - granted = READ_CONTROL | WRITE_DAC | FILE_READ_ATTRIBUTES | - DELETE; - - granted |= le32_to_cpu(ace->access_req); - - if (!pdacl->num_aces) - granted = GENERIC_ALL_FLAGS; - } - if (IS_ENABLED(CONFIG_FS_POSIX_ACL)) { posix_acls = get_inode_acl(d_inode(path->dentry), ACL_TYPE_ACCESS); - if (!IS_ERR_OR_NULL(posix_acls) && !found) { + if (!IS_ERR_OR_NULL(posix_acls) && !found && !others_ace) { unsigned int id = -1; pa_entry = posix_acls->a_entries; @@ -1583,19 +1605,27 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, } } - switch (ace->type) { - case ACCESS_ALLOWED_ACE_TYPE: - access_bits = le32_to_cpu(ace->access_req); - break; - case ACCESS_DENIED_ACE_TYPE: - case ACCESS_DENIED_CALLBACK_ACE_TYPE: - access_bits = le32_to_cpu(~ace->access_req); - break; + if (!(*pdaccess & FILE_MAXIMAL_ACCESS_LE)) { + switch (ace->type) { + case ACCESS_ALLOWED_ACE_TYPE: + access_bits = le32_to_cpu(ace->access_req); + break; + case ACCESS_DENIED_ACE_TYPE: + case ACCESS_DENIED_CALLBACK_ACE_TYPE: + access_bits = le32_to_cpu(~ace->access_req); + break; + } } check_access_bits: - if (granted & - ~(access_bits | FILE_READ_ATTRIBUTES | READ_CONTROL | WRITE_DAC | DELETE)) { + if (strict) { + access_bits &= granted; + } else { + access_bits |= FILE_READ_ATTRIBUTES | READ_CONTROL | + WRITE_DAC | DELETE; + } + + if (granted & ~access_bits) { ksmbd_debug(SMB, "Access denied with winACL, granted : %x, access_req : %x\n", granted, le32_to_cpu(ace->access_req)); rc = -EACCES; diff --git a/fs/smb/server/smbacl.h b/fs/smb/server/smbacl.h index ab21ba2cd4df35..01810c16cc0461 100644 --- a/fs/smb/server/smbacl.h +++ b/fs/smb/server/smbacl.h @@ -95,7 +95,8 @@ bool smb_inherit_flags(int flags, bool is_dir); int smb_inherit_dacl(struct ksmbd_conn *conn, const struct path *path, unsigned int uid, unsigned int gid); int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, - __le32 *pdaccess, int uid); + __le32 *pdaccess, __le32 raw_daccess, int uid, + bool strict); int set_info_sec(struct ksmbd_conn *conn, struct ksmbd_tree_connect *tcon, const struct path *path, struct smb_ntsd *pntsd, int ntsd_len, bool type_check, bool get_write); diff --git a/fs/smb/server/transport_ipc.c b/fs/smb/server/transport_ipc.c index 0c581b9624d3e0..5ad33fbfcbeff3 100644 --- a/fs/smb/server/transport_ipc.c +++ b/fs/smb/server/transport_ipc.c @@ -322,6 +322,15 @@ static int ipc_server_config_on_startup(struct ksmbd_startup_request *req) goto out; } server_conf.share_fake_fscaps = req->share_fake_fscaps; + + /* AAPL model string for Finder icon */ + if (req->aapl_model[0]) + strscpy(server_conf.aapl_model, req->aapl_model, + sizeof(server_conf.aapl_model)); + else + strscpy(server_conf.aapl_model, "Xserve", + sizeof(server_conf.aapl_model)); + ksmbd_init_domain(req->sub_auth); if (req->smb2_max_read) @@ -497,6 +506,9 @@ static int ipc_validate_msg(struct ipc_msg_table_entry *entry) { struct ksmbd_rpc_command *resp = entry->response; + if (entry->msg_sz < sizeof(struct ksmbd_rpc_command)) + return -EINVAL; + if (check_add_overflow(sizeof(struct ksmbd_rpc_command), resp->payload_sz, &msg_sz)) return -EINVAL; @@ -506,6 +518,9 @@ static int ipc_validate_msg(struct ipc_msg_table_entry *entry) { struct ksmbd_spnego_authen_response *resp = entry->response; + if (entry->msg_sz < sizeof(struct ksmbd_spnego_authen_response)) + return -EINVAL; + msg_sz = sizeof(struct ksmbd_spnego_authen_response) + resp->session_key_len + resp->spnego_blob_len; break; @@ -514,20 +529,33 @@ static int ipc_validate_msg(struct ipc_msg_table_entry *entry) { struct ksmbd_share_config_response *resp = entry->response; - if (resp->payload_sz) { - if (resp->payload_sz < resp->veto_list_sz) - return -EINVAL; + if (entry->msg_sz < sizeof(struct ksmbd_share_config_response)) + return -EINVAL; - if (check_add_overflow(sizeof(struct ksmbd_share_config_response), - resp->payload_sz, &msg_sz)) - return -EINVAL; - } + if (strnlen(resp->share_name, sizeof(resp->share_name)) == + sizeof(resp->share_name)) + return -EINVAL; + + if (resp->veto_list_sz > resp->payload_sz) + return -EINVAL; + + if (resp->flags != KSMBD_SHARE_FLAG_INVALID && + !(resp->flags & KSMBD_SHARE_FLAG_PIPE) && + resp->payload_sz <= resp->veto_list_sz) + return -EINVAL; + + if (check_add_overflow(sizeof(struct ksmbd_share_config_response), + resp->payload_sz, &msg_sz)) + return -EINVAL; break; } case KSMBD_EVENT_LOGIN_REQUEST_EXT: { struct ksmbd_login_response_ext *resp = entry->response; + if (entry->msg_sz < sizeof(struct ksmbd_login_response_ext)) + return -EINVAL; + if (resp->ngroups) { if (resp->ngroups < 0 || resp->ngroups > NGROUPS_MAX) { diff --git a/fs/smb/server/transport_tcp.c b/fs/smb/server/transport_tcp.c index 13b711ea575d8e..990b14e5d3ea23 100644 --- a/fs/smb/server/transport_tcp.c +++ b/fs/smb/server/transport_tcp.c @@ -434,6 +434,11 @@ static void ksmbd_tcp_disconnect(struct ksmbd_transport *t) atomic_dec(&active_num_conn); } +static void ksmbd_tcp_shutdown(struct ksmbd_transport *t) +{ + kernel_sock_shutdown(TCP_TRANS(t)->sock, SHUT_RDWR); +} + static void tcp_destroy_socket(struct socket *ksmbd_socket) { int ret; @@ -680,5 +685,6 @@ static const struct ksmbd_transport_ops ksmbd_tcp_transport_ops = { .read = ksmbd_tcp_read, .writev = ksmbd_tcp_writev, .disconnect = ksmbd_tcp_disconnect, + .shutdown = ksmbd_tcp_shutdown, .free_transport = ksmbd_tcp_free_transport, }; diff --git a/fs/smb/server/vfs.c b/fs/smb/server/vfs.c index d324585c0566bd..233d127f270155 100644 --- a/fs/smb/server/vfs.c +++ b/fs/smb/server/vfs.c @@ -297,9 +297,6 @@ static int check_lock_range(struct file *filp, loff_t start, loff_t end, struct file_lock_context *ctx = locks_inode_context(file_inode(filp)); int error = 0; - if (start == end) - return 0; - if (!ctx || list_empty_careful(&ctx->flc_posix)) return 0; diff --git a/fs/smb/server/vfs_cache.c b/fs/smb/server/vfs_cache.c index 9bd8dce660afa1..293dab9b43be17 100644 --- a/fs/smb/server/vfs_cache.c +++ b/fs/smb/server/vfs_cache.c @@ -124,7 +124,8 @@ static int proc_show_files(struct seq_file *m, void *v) static int create_proc_files(void) { - ksmbd_proc_create("files", proc_show_files, NULL); + if (!ksmbd_proc_create("files", proc_show_files, NULL)) + return -ENOMEM; return 0; } #else @@ -497,6 +498,7 @@ static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) { struct file *filp; struct ksmbd_lock *smb_lock, *tmp_lock; + struct ksmbd_work *cn_work; fd_limit_close(); ksmbd_remove_durable_fd(fp); @@ -510,8 +512,9 @@ static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) if (!IS_ERR_OR_NULL(filp)) fput(filp); - /* because the reference count of fp is 0, it is guaranteed that - * there are not accesses to fp->lock_list. + /* + * The zero fp reference count serializes access to fp->lock_list, but + * the VFS may still have blocked requests chained below these locks. */ list_for_each_entry_safe(smb_lock, tmp_lock, &fp->lock_list, flist) { struct ksmbd_conn *conn = smb_lock->conn; @@ -524,11 +527,58 @@ static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) ksmbd_conn_put(conn); } - list_del(&smb_lock->flist); + list_del_init(&smb_lock->flist); + ksmbd_vfs_posix_lock_unblock(smb_lock->fl); locks_free_lock(smb_lock->fl); kfree(smb_lock); } + /* + * Complete any CHANGE_NOTIFY left pending on this handle now that + * it is closed. KSMBD never completes CHANGE_NOTIFY spontaneously + * (no real change-notification backend), only on close -- matching + * genuine SMB2/macOS smbfs semantics and avoiding the Finder + * "directory changed, re-enumerate everything" loop. + * + * smb2_notify() on another connection can be adding to + * notify_pendings under fp->f_lock at the same time this handle is + * closed, and a client-sent CANCEL can concurrently be racing to + * claim the same entry via smb2_notify_cancel_fn() (smb2pdu.c). + * Pop one entry at a time under the lock via list_del_init() rather + * than a bulk list_splice_init(): list_del_init() leaves the node + * self-linked ("empty"), which is what the cancel path checks under + * the same lock to tell whether it lost the race -- a bulk splice + * would instead relink every entry into a shared local list, so an + * entry claimed here would still read as "not empty" to a racing + * cancel_fn, and both sides could end up freeing the same work. + * ksmbd_conn_write() can sleep (it takes conn's write mutex), so it + * must not be called while fp->f_lock is held -- release the lock + * before processing each popped entry, then reacquire it for the + * next. + */ + for (;;) { + spin_lock(&fp->f_lock); + if (list_empty(&fp->notify_pendings)) { + spin_unlock(&fp->f_lock); + break; + } + cn_work = list_first_entry(&fp->notify_pendings, + struct ksmbd_work, notify_entry); + list_del_init(&cn_work->notify_entry); + spin_unlock(&fp->f_lock); + + ksmbd_conn_write(cn_work); + /* + * release_async_work() removes cn_work from + * conn->async_requests, frees cancel_argv, and releases+zeroes + * async_id -- all needed before ksmbd_free_work_struct(), which + * only releases async_id itself if still nonzero (i.e. if this + * hadn't already been done). + */ + release_async_work(cn_work); + ksmbd_free_work_struct(cn_work); + } + /* * Drop fp's strong reference on conn (taken in ksmbd_open_fd() / * ksmbd_reopen_durable_fd()). Durable fps that reached the @@ -629,6 +679,11 @@ static struct ksmbd_file *ksmbd_fp_get(struct ksmbd_file *fp) return fp; } +struct ksmbd_file *ksmbd_file_get(struct ksmbd_file *fp) +{ + return ksmbd_fp_get(fp); +} + static struct ksmbd_file *__ksmbd_lookup_fd(struct ksmbd_file_table *ft, u64 id) { @@ -1050,6 +1105,7 @@ struct ksmbd_file *ksmbd_open_fd(struct ksmbd_work *work, struct file *filp) INIT_LIST_HEAD(&fp->blocked_works); INIT_LIST_HEAD(&fp->node); INIT_LIST_HEAD(&fp->lock_list); + INIT_LIST_HEAD(&fp->notify_pendings); spin_lock_init(&fp->f_lock); mutex_init(&fp->readdir_lock); atomic_set(&fp->refcount, 1); @@ -1446,9 +1502,12 @@ void ksmbd_launch_ksmbd_durable_scavenger(void) server_conf.dh_task = kthread_run(ksmbd_durable_scavenger, (void *)NULL, "ksmbd-durable-scavenger"); - if (IS_ERR(server_conf.dh_task)) + if (IS_ERR(server_conf.dh_task)) { pr_err("cannot start conn thread, err : %ld\n", PTR_ERR(server_conf.dh_task)); + server_conf.dh_task = NULL; + durable_scavenger_running = false; + } mutex_unlock(&durable_scavenger_lock); } @@ -1471,7 +1530,7 @@ void ksmbd_stop_durable_scavenger(void) } /* - * ksmbd_vfs_copy_durable_owner - Copy owner info for durable reconnect + * ksmbd_vfs_set_durable_owner - Store owner info for durable replay/reconnect * @fp: ksmbd file pointer to store owner info * @user: user pointer to copy from * @@ -1480,8 +1539,8 @@ void ksmbd_stop_durable_scavenger(void) * * Return: 0 on success, or negative error code on failure */ -static int ksmbd_vfs_copy_durable_owner(struct ksmbd_file *fp, - struct ksmbd_user *user) +int ksmbd_vfs_set_durable_owner(struct ksmbd_file *fp, + struct ksmbd_user *user) { char *name; @@ -1552,7 +1611,7 @@ static bool session_fd_check(struct ksmbd_tree_connect *tcon, if (WARN_ON_ONCE(!fp->conn)) return false; - if (ksmbd_vfs_copy_durable_owner(fp, user)) + if (ksmbd_vfs_set_durable_owner(fp, user)) return false; /* @@ -1620,7 +1679,8 @@ void ksmbd_close_session_fds(struct ksmbd_work *work) int ksmbd_init_global_file_table(void) { - create_proc_files(); + if (create_proc_files()) + pr_warn("Unable to create files procfs entry\n"); return ksmbd_init_file_table(&global_ft); } diff --git a/fs/smb/server/vfs_cache.h b/fs/smb/server/vfs_cache.h index b9e27307a26c15..8bb4396f952386 100644 --- a/fs/smb/server/vfs_cache.h +++ b/fs/smb/server/vfs_cache.h @@ -82,6 +82,13 @@ struct durable_owner { char *name; }; +#define KSMBD_LOCK_SEQ_ARRAY_SIZE 64 + +struct ksmbd_lock_sequence { + bool valid; + u8 sequence; +}; + struct ksmbd_file { struct file *filp; u64 persistent_id; @@ -101,6 +108,7 @@ struct ksmbd_file { __le32 saccess; __le32 coption; __le32 cdoption; + __le32 create_file_attributes; __u64 create_time; __u64 change_time; __u64 allocation_size; @@ -121,6 +129,8 @@ struct ksmbd_file { unsigned int durable_timeout; unsigned int durable_scavenger_timeout; + /* CREATE action returned when this durable handle was established. */ + __le32 create_action; /* if ls is happening on directory, below is valid*/ struct ksmbd_readdir_data readdir_data; @@ -132,9 +142,20 @@ struct ksmbd_file { bool is_persistent; bool is_resilient; bool durable_reconnect_disabled; + bool durable_replay_consumed; bool is_posix_ctxt; struct durable_owner owner; + __le16 channel_sequence; + unsigned int outstanding_requests; + unsigned int outstanding_pre_requests; + struct ksmbd_lock_sequence lock_seq[KSMBD_LOCK_SEQ_ARRAY_SIZE]; + + /* + * Pending CHANGE_NOTIFY completions for this handle, sent with + * STATUS_NOTIFY_CLEANUP when the handle is closed. + */ + struct list_head notify_pendings; }; static inline void set_ctx_actor(struct dir_context *ctx, @@ -167,6 +188,9 @@ struct ksmbd_file *ksmbd_lookup_fd_fast(struct ksmbd_work *work, u64 id); struct ksmbd_file *ksmbd_lookup_foreign_fd(struct ksmbd_work *work, u64 id); struct ksmbd_file *ksmbd_lookup_fd_slow(struct ksmbd_work *work, u64 id, u64 pid); +int ksmbd_vfs_set_durable_owner(struct ksmbd_file *fp, + struct ksmbd_user *user); +struct ksmbd_file *ksmbd_file_get(struct ksmbd_file *fp); void ksmbd_fd_put(struct ksmbd_work *work, struct ksmbd_file *fp); struct ksmbd_inode *ksmbd_inode_lookup_lock(struct dentry *d); void ksmbd_inode_put(struct ksmbd_inode *ci); diff --git a/fs/smb/smbdirect/accept.c b/fs/smb/smbdirect/accept.c index 5297400058385e..039c9bfbd8acda 100644 --- a/fs/smb/smbdirect/accept.c +++ b/fs/smb/smbdirect/accept.c @@ -145,11 +145,21 @@ int smbdirect_accept_connect_request(struct smbdirect_socket *sc, rdma_accept_failed: /* - * smbdirect_connection_destroy_qp() calls ib_drain_qp(), - * so that smbdirect_accept_negotiate_recv_done() will - * call smbdirect_connection_put_recv_io() + * The recv_io posted above is now owned by the QP (recv_io was set to + * NULL after a successful post). smbdirect_connection_destroy_qp() + * calls ib_drain_qp(), whose completion + * (smbdirect_accept_negotiate_recv_done) returns the recv_io to the + * free list via smbdirect_connection_put_recv_io(). It therefore MUST + * run BEFORE smbdirect_connection_destroy_mem_pools(): otherwise the + * posted recv_io is still outstanding when kmem_cache_destroy() runs + * ("Slab cache still has objects") and is later freed into an + * already-destroyed mempool (mempool_free_bulk NULL-ptr-deref). */ + smbdirect_connection_destroy_qp(sc); + smbdirect_connection_destroy_mem_pools(sc); + return ret; post_recv_io_failed: + /* post failed: recv_io was not accepted by the QP, still in hand */ if (recv_io) smbdirect_connection_put_recv_io(recv_io); get_recv_io_failed: diff --git a/fs/smb/smbdirect/connection.c b/fs/smb/smbdirect/connection.c index 8adf580975344b..fe99537207176e 100644 --- a/fs/smb/smbdirect/connection.c +++ b/fs/smb/smbdirect/connection.c @@ -403,12 +403,21 @@ void smbdirect_connection_destroy_qp(struct smbdirect_socket *sc) sc->ib.qp = NULL; rdma_destroy_qp(sc->rdma.cm_id); } + /* + * These CQs were created with ib_alloc_cq_any(), which arms an internal + * completion handler (ib_cq_poll_work for IB_POLL_WORKQUEUE). They MUST be + * torn down with ib_free_cq(), which cancel_work_sync()es that poll work + * before freeing the CQ. ib_destroy_cq() skips that step, so a completion + * posted late by the (software) provider — e.g. rxe posting an RNR error + * from rxe_receiver after rdma_destroy_qp() — re-queues ib_cq_poll_work on + * an already-freed CQ (KASAN slab-use-after-free in ib_cq_poll_work). + */ if (sc->ib.recv_cq) { - ib_destroy_cq(sc->ib.recv_cq); + ib_free_cq(sc->ib.recv_cq); sc->ib.recv_cq = NULL; } if (sc->ib.send_cq) { - ib_destroy_cq(sc->ib.send_cq); + ib_free_cq(sc->ib.send_cq); sc->ib.send_cq = NULL; } if (sc->ib.pd) { diff --git a/fs/smb/smbdirect/socket.c b/fs/smb/smbdirect/socket.c index 39cca7219c4df8..bb02df6158b97e 100644 --- a/fs/smb/smbdirect/socket.c +++ b/fs/smb/smbdirect/socket.c @@ -305,12 +305,26 @@ void __smbdirect_socket_schedule_cleanup(struct smbdirect_socket *sc, * disconnect all pending and ready sockets * * First we move ready sockets to pending again. + * + * Only a socket that was a listener (listen.backlog != -1) owns a + * populated listen.ready/pending list. Guarding on that also keeps + * lockdep quiet: without it, the listener holds sc->listen.lock while + * the loop recurses into each child psc, which takes psc->listen.lock. + * Those are always different instances of the same lock class -- a + * child never listens, so the nesting is strictly listener -> child + * and cannot really deadlock -- but lockdep only sees the class and + * reports "possible recursive locking". A child has empty listen + * lists and nothing to do here, so skipping it loses nothing, and a + * pending child stays on its listener's list for the free path + * (smbdirect_socket_destroy) to reap. */ - spin_lock_irqsave(&sc->listen.lock, flags); - list_splice_init(&sc->listen.ready, &sc->listen.pending); - list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list) - smbdirect_socket_schedule_cleanup(psc, sc->first_error); - spin_unlock_irqrestore(&sc->listen.lock, flags); + if (sc->listen.backlog != -1) { /* was a listener */ + spin_lock_irqsave(&sc->listen.lock, flags); + list_splice_init(&sc->listen.ready, &sc->listen.pending); + list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list) + smbdirect_socket_schedule_cleanup(psc, sc->first_error); + spin_unlock_irqrestore(&sc->listen.lock, flags); + } switch (sc->status) { case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: @@ -405,12 +419,20 @@ static void smbdirect_socket_cleanup_work(struct work_struct *work) * disconnect all pending and ready sockets * * First we move ready sockets to pending again. + * + * Guarded on listen.backlog != -1 for the same reason as in + * __smbdirect_socket_schedule_cleanup(): only a listener owns a + * populated listen list, and skipping the block for a child avoids + * nesting psc->listen.lock under a listener's listen.lock (different + * instances of one class -- harmless, but lockdep cannot tell). */ - spin_lock_irqsave(&sc->listen.lock, flags); - list_splice_init(&sc->listen.ready, &sc->listen.pending); - list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list) - smbdirect_socket_schedule_cleanup(psc, sc->first_error); - spin_unlock_irqrestore(&sc->listen.lock, flags); + if (sc->listen.backlog != -1) { /* was a listener */ + spin_lock_irqsave(&sc->listen.lock, flags); + list_splice_init(&sc->listen.ready, &sc->listen.pending); + list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list) + smbdirect_socket_schedule_cleanup(psc, sc->first_error); + spin_unlock_irqrestore(&sc->listen.lock, flags); + } switch (sc->status) { case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: @@ -473,6 +495,7 @@ static void smbdirect_socket_destroy(struct smbdirect_socket *sc) struct smbdirect_recv_io *recv_io; struct smbdirect_recv_io *recv_tmp; LIST_HEAD(all_list); + LIST_HEAD(pending_list); unsigned long flags; smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, @@ -530,24 +553,29 @@ static void smbdirect_socket_destroy(struct smbdirect_socket *sc) * disconnect all pending and ready sockets * * We move ready sockets to pending again. + * + * Capture them here -- rdma_lock_handler(sc->rdma.cm_id) is held above, + * so a concurrent CM CONNECT_REQUEST cannot add more; sc->listen.lock + * below only protects the list splice itself -- but DEFER releasing + * them until the listener's cm_id is destroyed: + * + * - smbdirect_socket_release() -> smbdirect_socket_destroy() takes the + * child's own rdma_lock_handler() lock (&id_priv->handler_mutex). + * The listener's and the child's cm_id are always different + * instances, so the nesting cannot really deadlock, but lockdep only + * sees one lock class and reports "possible recursive locking". + * + * - rdma_destroy_id() of a child before the listener's own + * rdma_destroy_id() below lets _cma_cancel_listens() walk the freed + * child id_priv (KASAN slab-use-after-free in __mutex_lock()). + * + * The children are independent sockets whose teardown does not need + * the listener's handler lock. */ spin_lock_irqsave(&sc->listen.lock, flags); - list_splice_tail_init(&sc->listen.ready, &all_list); - list_splice_tail_init(&sc->listen.pending, &all_list); + list_splice_tail_init(&sc->listen.ready, &pending_list); + list_splice_tail_init(&sc->listen.pending, &pending_list); spin_unlock_irqrestore(&sc->listen.lock, flags); - psockets = list_count_nodes(&all_list); - if (sc->listen.backlog != -1) /* was a listener */ - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "release %zu pending sockets\n", psockets); - list_for_each_entry_safe(psc, tsc, &all_list, accept.list) { - list_del_init(&psc->accept.list); - psc->accept.listener = NULL; - smbdirect_socket_release(psc); - } - if (sc->listen.backlog != -1) /* was a listener */ - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "released %zu pending sockets\n", psockets); - INIT_LIST_HEAD(&all_list); /* It's not possible for upper layer to get to reassembly */ if (sc->listen.backlog == -1) /* was not a listener */ @@ -577,6 +605,26 @@ static void smbdirect_socket_destroy(struct smbdirect_socket *sc) sc->rdma.cm_id = NULL; } + /* + * The listener's rdma_lock_handler() lock is dropped and its cm_id is + * destroyed, so it is safe to release the child sockets captured + * above: each release recurses into smbdirect_socket_destroy() and + * takes that child's own handler_mutex without nesting it under the + * listener's, and _cma_cancel_listens() can no longer reach them. + */ + psockets = list_count_nodes(&pending_list); + if (sc->listen.backlog != -1) /* was a listener */ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "release %zu pending sockets\n", psockets); + list_for_each_entry_safe(psc, tsc, &pending_list, accept.list) { + list_del_init(&psc->accept.list); + psc->accept.listener = NULL; + smbdirect_socket_release(psc); + } + if (sc->listen.backlog != -1) /* was a listener */ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "released %zu pending sockets\n", psockets); + if (sc->listen.backlog == -1) /* was not a listener */ smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, "destroying mem pools\n"); diff --git a/fs/squashfs/cache.c b/fs/squashfs/cache.c index 67abd4dff22223..2807b80d46b721 100644 --- a/fs/squashfs/cache.c +++ b/fs/squashfs/cache.c @@ -299,7 +299,7 @@ int squashfs_copy_data(void *buffer, struct squashfs_cache_entry *entry, { int remaining = length; - if (length == 0) + if (length == 0 || offset < 0) return 0; else if (buffer == NULL) return min(length, entry->length - offset); diff --git a/fs/stat.c b/fs/stat.c index 89909746bed19e..c461c305423408 100644 --- a/fs/stat.c +++ b/fs/stat.c @@ -53,7 +53,7 @@ void fill_mg_cmtime(struct kstat *stat, u32 request_mask, struct inode *inode) } stat->mtime = inode_get_mtime(inode); - stat->ctime.tv_sec = inode->i_ctime_sec; + stat->ctime.tv_sec = inode_get_ctime_sec(inode); stat->ctime.tv_nsec = (u32)atomic_read(pcn); if (!(stat->ctime.tv_nsec & I_CTIME_QUERIED)) stat->ctime.tv_nsec = ((u32)atomic_fetch_or(I_CTIME_QUERIED, pcn)); diff --git a/fs/super.c b/fs/super.c index ffdcc6a2e0debd..d94f3a3e988df7 100644 --- a/fs/super.c +++ b/fs/super.c @@ -2018,7 +2018,7 @@ int freeze_super(struct super_block *sb, enum freeze_holder who, const void *fre int ret; if (!super_lock_excl(sb)) { - WARN_ON_ONCE("Dying superblock while freezing!"); + WARN_ONCE(1, "Dying superblock while freezing!"); return -EINVAL; } atomic_inc(&sb->s_active); @@ -2182,7 +2182,7 @@ int thaw_super(struct super_block *sb, enum freeze_holder who, const void *freeze_owner) { if (!super_lock_excl(sb)) { - WARN_ON_ONCE("Dying superblock while thawing!"); + WARN_ONCE(1, "Dying superblock while thawing!"); return -EINVAL; } return thaw_super_locked(sb, who, freeze_owner); diff --git a/fs/udf/inode.c b/fs/udf/inode.c index c519644acfd7de..e1fdf9f9c9f50d 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -336,65 +336,6 @@ const struct address_space_operations udf_aops = { .migrate_folio = buffer_migrate_folio, }; -/* - * Expand file stored in ICB to a normal one-block-file - * - * This function requires i_mutex held - */ -int udf_expand_file_adinicb(struct inode *inode) -{ - struct folio *folio; - struct udf_inode_info *iinfo = UDF_I(inode); - int err; - - WARN_ON_ONCE(!inode_is_locked(inode)); - if (!iinfo->i_lenAlloc) { - down_write(&iinfo->i_data_sem); - if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) - iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; - else - iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; - up_write(&iinfo->i_data_sem); - mark_inode_dirty(inode); - return 0; - } - - folio = __filemap_get_folio(inode->i_mapping, 0, - FGP_LOCK | FGP_ACCESSED | FGP_CREAT, GFP_KERNEL); - if (IS_ERR(folio)) - return PTR_ERR(folio); - - if (!folio_test_uptodate(folio)) - udf_adinicb_read_folio(folio); - down_write(&iinfo->i_data_sem); - memset(iinfo->i_data + iinfo->i_lenEAttr, 0x00, - iinfo->i_lenAlloc); - iinfo->i_lenAlloc = 0; - if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) - iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; - else - iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; - folio_mark_dirty(folio); - folio_unlock(folio); - up_write(&iinfo->i_data_sem); - err = filemap_fdatawrite(inode->i_mapping); - if (err) { - /* Restore everything back so that we don't lose data... */ - folio_lock(folio); - down_write(&iinfo->i_data_sem); - memcpy_from_folio(iinfo->i_data + iinfo->i_lenEAttr, - folio, 0, inode->i_size); - folio_unlock(folio); - iinfo->i_alloc_type = ICBTAG_FLAG_AD_IN_ICB; - iinfo->i_lenAlloc = inode->i_size; - up_write(&iinfo->i_data_sem); - } - folio_put(folio); - mark_inode_dirty(inode); - - return err; -} - #define UDF_MAP_CREATE 0x01 /* Mapping can allocate new blocks */ #define UDF_MAP_NOPREALLOC 0x02 /* Do not preallocate blocks */ @@ -455,6 +396,76 @@ out_read: return ret; } +/* + * Expand file stored in ICB to a normal one-block-file + * + * This function requires i_mutex held + */ +int udf_expand_file_adinicb(struct inode *inode) +{ + struct folio *folio; + struct udf_inode_info *iinfo = UDF_I(inode); + struct udf_map_rq map = { + .lblk = 0, + .iflags = UDF_MAP_CREATE, + }; + int err; + + WARN_ON_ONCE(!inode_is_locked(inode)); + if (!iinfo->i_lenAlloc) { + down_write(&iinfo->i_data_sem); + if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) + iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; + else + iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; + up_write(&iinfo->i_data_sem); + mark_inode_dirty(inode); + return 0; + } + + folio = __filemap_get_folio(inode->i_mapping, 0, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, GFP_KERNEL); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + if (!folio_test_uptodate(folio)) + udf_adinicb_read_folio(folio); + down_write(&iinfo->i_data_sem); + memset(iinfo->i_data + iinfo->i_lenEAttr, 0x00, + iinfo->i_lenAlloc); + iinfo->i_lenAlloc = 0; + if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) + iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; + else + iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; + up_write(&iinfo->i_data_sem); + + /* Allocate the block underlying the data */ + err = udf_map_block(inode, &map); + if (err < 0) + goto restore; + + folio_mark_dirty(folio); + folio_unlock(folio); + err = filemap_fdatawrite(inode->i_mapping); + if (err) { + /* Restore everything back so that we don't lose data... */ + folio_lock(folio); +restore: + down_write(&iinfo->i_data_sem); + memcpy_from_folio(iinfo->i_data + iinfo->i_lenEAttr, + folio, 0, inode->i_size); + iinfo->i_alloc_type = ICBTAG_FLAG_AD_IN_ICB; + iinfo->i_lenAlloc = inode->i_size; + up_write(&iinfo->i_data_sem); + folio_unlock(folio); + } + folio_put(folio); + mark_inode_dirty(inode); + + return err; +} + static int __udf_get_block(struct inode *inode, sector_t block, struct buffer_head *bh_result, int flags) { diff --git a/fs/udf/super.c b/fs/udf/super.c index 7b85f5a2b79fc4..9686078bba6491 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -2054,6 +2054,17 @@ static int udf_load_vrs(struct super_block *sb, struct udf_options *uopt, return 0; } +static void udf_mark_buffer_dirty(struct buffer_head *bh) +{ + /* + * We set buffer uptodate unconditionally here to avoid spurious + * warnings from mark_buffer_dirty() when previous EIO has marked + * the buffer as !uptodate + */ + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); +} + static void udf_finalize_lvid(struct logicalVolIntegrityDesc *lvid) { struct timespec64 ts; @@ -2089,7 +2100,7 @@ static void udf_open_lvid(struct super_block *sb) UDF_SET_FLAG(sb, UDF_FLAG_INCONSISTENT); udf_finalize_lvid(lvid); - mark_buffer_dirty(bh); + udf_mark_buffer_dirty(bh); sbi->s_lvid_dirty = 0; mutex_unlock(&sbi->s_alloc_mutex); /* Make opening of filesystem visible on the media immediately */ @@ -2122,14 +2133,8 @@ static void udf_close_lvid(struct super_block *sb) if (!UDF_QUERY_FLAG(sb, UDF_FLAG_INCONSISTENT)) lvid->integrityType = cpu_to_le32(LVID_INTEGRITY_TYPE_CLOSE); - /* - * We set buffer uptodate unconditionally here to avoid spurious - * warnings from mark_buffer_dirty() when previous EIO has marked - * the buffer as !uptodate - */ - set_buffer_uptodate(bh); udf_finalize_lvid(lvid); - mark_buffer_dirty(bh); + udf_mark_buffer_dirty(bh); sbi->s_lvid_dirty = 0; mutex_unlock(&sbi->s_alloc_mutex); /* Make closing of filesystem visible on the media immediately */ @@ -2411,7 +2416,7 @@ static int udf_sync_fs(struct super_block *sb, int wait) * Blockdevice will be synced later so we don't have to submit * the buffer for IO */ - mark_buffer_dirty(bh); + udf_mark_buffer_dirty(bh); sbi->s_lvid_dirty = 0; } mutex_unlock(&sbi->s_alloc_mutex); diff --git a/fs/udf/symlink.c b/fs/udf/symlink.c index fe03745d09b181..a05d1888a2babe 100644 --- a/fs/udf/symlink.c +++ b/fs/udf/symlink.c @@ -36,6 +36,8 @@ static int udf_pc_to_char(struct super_block *sb, unsigned char *from, /* Reserve one byte for terminating \0 */ tolen--; while (elen < fromlen) { + if (fromlen - elen < sizeof(struct pathComponent)) + return -EIO; pc = (struct pathComponent *)(from + elen); elen += sizeof(struct pathComponent); switch (pc->componentType) { diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h index 714d111d80530f..61782dfdfe5a5e 100644 --- a/include/acpi/acpi_bus.h +++ b/include/acpi/acpi_bus.h @@ -686,6 +686,7 @@ void acpi_bus_unregister_driver(struct acpi_driver *driver); int acpi_bus_scan(acpi_handle handle); void acpi_bus_trim(struct acpi_device *start); acpi_status acpi_bus_get_ejd(acpi_handle handle, acpi_handle * ejd); +struct device *acpi_bus_get_primary_device(struct acpi_device *adev); int acpi_match_device_ids(struct acpi_device *device, const struct acpi_device_id *ids); void acpi_set_modalias(struct acpi_device *adev, const char *default_id, @@ -1000,6 +1001,11 @@ int acpi_scan_add_dep(acpi_handle handle, struct acpi_handle_list *dep_devices); u32 arch_acpi_add_auto_dep(acpi_handle handle); #else /* CONFIG_ACPI */ +static inline struct device *acpi_bus_get_primary_device(struct acpi_device *adev) +{ + return NULL; +} + static inline bool acpi_of_match_device(const struct acpi_device *adev, const struct of_device_id *of_match_table, const struct of_device_id **of_id) diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 5659f4b5a1252f..ee9c5d354a8567 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -839,12 +839,20 @@ .stab.index 0 : { *(.stab.index) } \ .stab.indexstr 0 : { *(.stab.indexstr) } +#ifdef CONFIG_KLP_BUILD +#define KLP_SYMID \ + .klp.symid 0 : { *(.klp.symid) } +#else +#define KLP_SYMID +#endif + /* Required sections not related to debugging. */ #define ELF_DETAILS \ .comment 0 : { *(.comment) } \ .symtab 0 : { *(.symtab) } \ .strtab 0 : { *(.strtab) } \ - .shstrtab 0 : { *(.shstrtab) } + .shstrtab 0 : { *(.shstrtab) } \ + KLP_SYMID #define MODINFO \ .modinfo : { *(.modinfo) . = ALIGN(8); } diff --git a/include/clocksource/samsung_pwm.h b/include/clocksource/samsung_pwm.h index 9b435caa95fef5..36f6f246e55940 100644 --- a/include/clocksource/samsung_pwm.h +++ b/include/clocksource/samsung_pwm.h @@ -15,7 +15,7 @@ * spinlock is not shared between both drivers. */ #ifdef CONFIG_CLKSRC_SAMSUNG_PWM -extern spinlock_t samsung_pwm_lock; +extern raw_spinlock_t samsung_pwm_lock; #endif struct samsung_pwm_variant { diff --git a/include/cxl/mailbox.h b/include/cxl/mailbox.h index c4e99e2e3a9d47..d008b9db07aa6c 100644 --- a/include/cxl/mailbox.h +++ b/include/cxl/mailbox.h @@ -50,6 +50,7 @@ struct cxl_mbox_cmd { * @payload_size: Size of space for payload * (CXL 3.1 8.2.8.4.3 Mailbox Capabilities Register) * @mbox_mutex: mutex protects device mailbox and firmware + * @feat_mutex: serializes multi-part Get/Set Feature transfers * @mbox_wait: rcuwait for mailbox * @mbox_send: @dev specific transport for transmitting mailbox commands * @feat_cap: Features capability @@ -60,6 +61,7 @@ struct cxl_mailbox { DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX); size_t payload_size; struct mutex mbox_mutex; /* lock to protect mailbox context */ + struct mutex feat_mutex; struct rcuwait mbox_wait; int (*mbox_send)(struct cxl_mailbox *cxl_mbox, struct cxl_mbox_cmd *cmd); enum cxl_features_capability feat_cap; diff --git a/include/drm/bridge/dw_dp.h b/include/drm/bridge/dw_dp.h index 25363541e69d51..22105c3e8e4d66 100644 --- a/include/drm/bridge/dw_dp.h +++ b/include/drm/bridge/dw_dp.h @@ -24,4 +24,5 @@ struct dw_dp_plat_data { struct dw_dp *dw_dp_bind(struct device *dev, struct drm_encoder *encoder, const struct dw_dp_plat_data *plat_data); +void dw_dp_unbind(struct dw_dp *dp); #endif /* __DW_DP__ */ diff --git a/include/linux/bio-integrity.h b/include/linux/bio-integrity.h index c3dda32fd80306..0ea2a8bf7efb73 100644 --- a/include/linux/bio-integrity.h +++ b/include/linux/bio-integrity.h @@ -17,6 +17,9 @@ enum bip_flags { BIP_MEMPOOL = 1 << 15, /* buffer backed by mempool */ }; +/* flags that require generate/verify action. */ +#define BIP_CHECK_FLAGS (BIP_CHECK_GUARD | BIP_CHECK_REFTAG | BIP_CHECK_APPTAG) + struct bio_integrity_payload { struct bvec_iter bip_iter; diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 7719f652844560..77e2075f77c737 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -570,7 +570,7 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, if (IS_ERR_OR_NULL(rec)) { if (long_memcpy) - bpf_long_memcpy(dst, src, round_up(size, 8)); + bpf_long_memcpy(dst, src, size); else memcpy(dst, src, size); return; @@ -593,7 +593,7 @@ static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src) { - bpf_obj_memcpy(map->record, dst, src, map->value_size, true); + bpf_obj_memcpy(map->record, dst, src, round_up(map->value_size, 8), true); } static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src) @@ -1118,21 +1118,6 @@ static inline bool bpf_pseudo_func(const struct bpf_insn *insn) return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC; } -/* Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an - * atomic load or store, and false if it is a read-modify-write instruction. - */ -static inline bool -bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn) -{ - switch (atomic_insn->imm) { - case BPF_LOAD_ACQ: - case BPF_STORE_REL: - return true; - default: - return false; - } -} - struct bpf_prog_ops { int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); @@ -1524,6 +1509,7 @@ int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids, struct bpf_tracing_multi_link *link); int bpf_trampoline_multi_detach(struct bpf_prog *prog, struct bpf_tracing_multi_link *link); +void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags); /* * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn @@ -1647,6 +1633,7 @@ static inline int bpf_trampoline_multi_detach(struct bpf_prog *prog, { return -ENOTSUPP; } +static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) {} #endif struct bpf_func_info_aux { @@ -1865,8 +1852,9 @@ struct bpf_prog_aux { struct bpf_prog { u16 pages; /* Number of allocated pages */ - u16 jited:1, /* Is our filter JIT'ed? */ + u32 jited:1, /* Is our filter JIT'ed? */ jit_requested:1,/* archs need to JIT the prog */ + jit_required:1, /* program strictly requires JIT compiler */ gpl_compatible:1, /* Is filter GPL compatible? */ cb_access:1, /* Is control block accessed? */ dst_needed:1, /* Do we need dst entry? */ @@ -3146,7 +3134,7 @@ int btf_struct_access(struct bpf_verifier_log *log, bool btf_struct_ids_match(struct bpf_verifier_log *log, const struct btf *btf, u32 id, int off, const struct btf *need_btf, u32 need_type_id, - bool strict); + bool strict, bool walk_flex_arrays); int btf_distill_func_proto(struct bpf_verifier_log *log, struct btf *btf, @@ -3170,7 +3158,6 @@ const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); void bpf_task_storage_free(struct task_struct *task); void bpf_cgrp_storage_free(struct cgroup *cgroup); -bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog); const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn); @@ -3509,11 +3496,6 @@ static inline void bpf_task_storage_free(struct task_struct *task) { } -static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) -{ - return false; -} - static inline const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn) @@ -4169,7 +4151,7 @@ bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image) } #endif -static inline bool bpf_map_supports_cpu_flags(enum bpf_map_type map_type) +static inline bool bpf_map_is_percpu_map(enum bpf_map_type map_type) { switch (map_type) { case BPF_MAP_TYPE_PERCPU_ARRAY: @@ -4196,7 +4178,7 @@ static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 all return -EINVAL; if (flags & (BPF_F_CPU | BPF_F_ALL_CPUS)) { - if (!bpf_map_supports_cpu_flags(map->map_type)) + if (!bpf_map_is_percpu_map(map->map_type)) return -EINVAL; if ((flags & BPF_F_CPU) && (flags & BPF_F_ALL_CPUS)) return -EINVAL; diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index 65d76a38974bac..bf3999daa080a1 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h @@ -273,6 +273,7 @@ static inline void dl_rebuild_rd_accounting(void) static inline void rebuild_sched_domains(void) { + guard(cpus_read_lock)(); partition_sched_domains(1, NULL, NULL); } diff --git a/include/linux/efi.h b/include/linux/efi.h index ccbc35479684a0..24221a8424121b 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -1212,8 +1212,8 @@ efi_call_acpi_prm_handler(efi_status_t (__efiapi *handler_addr)(u64, void *), /* * efi_runtime_service() function identifiers. - * "NONE" is used by efi_recover_from_page_fault() to check if the page - * fault happened while executing an efi runtime service. + * "NONE" is used by efi_crash_gracefully_on_page_fault() to check if the + * page fault happened while executing an efi runtime service. */ enum efi_rts_ids { EFI_NONE, diff --git a/include/linux/filter.h b/include/linux/filter.h index 14acb2455746fc..e0e83bf33fd1bc 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -383,6 +383,61 @@ static inline bool insn_is_cast_user(const struct bpf_insn *insn) /* Legacy alias */ #define BPF_STX_XADD(SIZE, DST, SRC, OFF) BPF_ATOMIC_OP(SIZE, BPF_ADD, DST, SRC, OFF) +/* + * Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an + * atomic load or store, and false if it is a read-modify-write instruction. + */ +static inline bool +bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn) +{ + switch (atomic_insn->imm) { + case BPF_LOAD_ACQ: + case BPF_STORE_REL: + return true; + default: + return false; + } +} + +/* + * A load-acquire is the only BPF_STX class instruction that reads into + * dst_reg from src_reg + off16, i.e. it has the operand roles of a BPF_LDX. + * Unlike bpf_atomic_is_load_store(), @insn is not assumed to be a BPF_ATOMIC + * instruction here, so that callers which walk all instruction classes can + * use this directly. + */ +static inline bool bpf_atomic_is_load_acq(const struct bpf_insn *insn) +{ + return BPF_CLASS(insn->code) == BPF_STX && + (BPF_MODE(insn->code) == BPF_ATOMIC || + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) && + insn->imm == BPF_LOAD_ACQ; +} + +/* + * Given an instruction @insn, return the number of the BPF register that a + * BPF_ATOMIC reads the value at its memory operand into, or -1 if there is + * no such register. That is the register a BPF_PROBE_ATOMIC has to clear when + * the access faults. Like bpf_atomic_is_load_acq(), @insn is not assumed to + * be a BPF_ATOMIC here. + */ +static inline int bpf_atomic_load_reg(const struct bpf_insn *insn) +{ + if (BPF_CLASS(insn->code) != BPF_STX || + (BPF_MODE(insn->code) != BPF_ATOMIC && + BPF_MODE(insn->code) != BPF_PROBE_ATOMIC)) + return -1; + + switch (insn->imm) { + case BPF_LOAD_ACQ: + return insn->dst_reg; + case BPF_CMPXCHG: + return BPF_REG_0; + default: + return (insn->imm & BPF_FETCH) ? insn->src_reg : -1; + } +} + /* Memory store, *(uint *) (dst_reg + off16) = imm32 */ #define BPF_ST_MEM(SIZE, DST, OFF, IMM) \ diff --git a/include/linux/fs.h b/include/linux/fs.h index 50ce731a2b78f1..09de7bf6f1d28e 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1598,12 +1598,12 @@ struct timespec64 inode_set_ctime_deleg(struct inode *inode, static inline time64_t inode_get_atime_sec(const struct inode *inode) { - return inode->i_atime_sec; + return READ_ONCE(inode->i_atime_sec); } static inline long inode_get_atime_nsec(const struct inode *inode) { - return inode->i_atime_nsec; + return READ_ONCE(inode->i_atime_nsec); } static inline struct timespec64 inode_get_atime(const struct inode *inode) @@ -1617,8 +1617,8 @@ static inline struct timespec64 inode_get_atime(const struct inode *inode) static inline struct timespec64 inode_set_atime_to_ts(struct inode *inode, struct timespec64 ts) { - inode->i_atime_sec = ts.tv_sec; - inode->i_atime_nsec = ts.tv_nsec; + WRITE_ONCE(inode->i_atime_sec, ts.tv_sec); + WRITE_ONCE(inode->i_atime_nsec, ts.tv_nsec); return ts; } @@ -1633,12 +1633,12 @@ static inline struct timespec64 inode_set_atime(struct inode *inode, static inline time64_t inode_get_mtime_sec(const struct inode *inode) { - return inode->i_mtime_sec; + return READ_ONCE(inode->i_mtime_sec); } static inline long inode_get_mtime_nsec(const struct inode *inode) { - return inode->i_mtime_nsec; + return READ_ONCE(inode->i_mtime_nsec); } static inline struct timespec64 inode_get_mtime(const struct inode *inode) @@ -1651,8 +1651,8 @@ static inline struct timespec64 inode_get_mtime(const struct inode *inode) static inline struct timespec64 inode_set_mtime_to_ts(struct inode *inode, struct timespec64 ts) { - inode->i_mtime_sec = ts.tv_sec; - inode->i_mtime_nsec = ts.tv_nsec; + WRITE_ONCE(inode->i_mtime_sec, ts.tv_sec); + WRITE_ONCE(inode->i_mtime_nsec, ts.tv_nsec); return ts; } @@ -1677,12 +1677,12 @@ static inline struct timespec64 inode_set_mtime(struct inode *inode, static inline time64_t inode_get_ctime_sec(const struct inode *inode) { - return inode->i_ctime_sec; + return READ_ONCE(inode->i_ctime_sec); } static inline long inode_get_ctime_nsec(const struct inode *inode) { - return inode->i_ctime_nsec & ~I_CTIME_QUERIED; + return READ_ONCE(inode->i_ctime_nsec) & ~I_CTIME_QUERIED; } static inline struct timespec64 inode_get_ctime(const struct inode *inode) diff --git a/include/linux/fsl/ntmp.h b/include/linux/fsl/ntmp.h index d3b6c476b91a2d..764ef28926085e 100644 --- a/include/linux/fsl/ntmp.h +++ b/include/linux/fsl/ntmp.h @@ -75,8 +75,10 @@ struct ntmp_user { /* NTMP table bitmaps for resource management */ u32 ett_bitmap_size; u32 ect_bitmap_size; + u16 maft_num_entries; unsigned long *ett_gid_bitmap; /* only valid for switch */ unsigned long *ect_gid_bitmap; /* only valid for switch */ + unsigned long *maft_eid_bitmap; /* only valid for ENETC */ }; struct maft_entry_data { diff --git a/include/linux/i3c/master.h b/include/linux/i3c/master.h index 4d2a6879332400..2edf40b112ae01 100644 --- a/include/linux/i3c/master.h +++ b/include/linux/i3c/master.h @@ -228,6 +228,8 @@ struct i3c_dev_desc { * every time the I3C device is rediscovered with a different dynamic * address assigned * @bus: I3C bus this device is attached to + * @node: unregistered device list node, only for use by + * i3c_master_register_new_i3c_devs(), it is not protected by a lock * * I3C device object exposed to I3C device drivers. The takes care of linking * this object to the relevant &struct_i3c_dev_desc one. @@ -238,6 +240,7 @@ struct i3c_device { struct device dev; struct i3c_dev_desc *desc; struct i3c_bus *bus; + struct list_head node; }; /* diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 56b43d594e6e29..12eb908380a2d1 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -212,24 +212,36 @@ struct iomap_write_ops { #define IOMAP_ATOMIC (1 << 9) /* torn-write protection */ #define IOMAP_DONTCACHE (1 << 10) -struct iomap_ops { - /* - * Return the existing mapping at pos, or reserve space starting at - * pos for up to length, as long as we can do it as a single mapping. - * The actual length is returned in iomap->length. - */ - int (*iomap_begin)(struct inode *inode, loff_t pos, loff_t length, - unsigned flags, struct iomap *iomap, - struct iomap *srcmap); +/* + * Return the existing mapping at pos, or reserve space starting at pos for up + * to length, as long as we can do it as a single mapping. + * The actual length is returned in iomap->length. + */ +typedef int (*iomap_iter_begin_fn)(struct inode *inode, loff_t pos, + loff_t length, unsigned flags, struct iomap *iomap, + struct iomap *srcmap); - /* - * Commit and/or unreserve space previous allocated using iomap_begin. - * Written indicates the length of the successful write operation which - * needs to be commited, while the rest needs to be unreserved. - * Written might be zero if no data was written. - */ - int (*iomap_end)(struct inode *inode, loff_t pos, loff_t length, - ssize_t written, unsigned flags, struct iomap *iomap); +/* + * Commit and/or unreserve space previously allocated by iomap_iter_begin_fn. + * Written indicates the length of the successful write operation which needs + * to be committed, while the rest needs to be unreserved. + * Written might be zero if no data was written. + */ +typedef int (*iomap_iter_end_fn)(struct inode *inode, loff_t pos, loff_t length, + ssize_t written, unsigned flags, struct iomap *iomap); + +/* + * Produce the next mapping (finishing the previous one if needed). + * Return 1 to continue iterating, 0 if the range is fully consumed, or a + * negative error on failure. + */ +typedef int (*iomap_iter_next_fn)(const struct iomap_iter *iter, + struct iomap *iomap, struct iomap *srcmap); + +struct iomap_ops { + iomap_iter_begin_fn iomap_begin; + iomap_iter_end_fn iomap_end; + iomap_iter_next_fn iomap_next; }; /** @@ -317,6 +329,71 @@ static inline const struct iomap *iomap_iter_srcmap(const struct iomap_iter *i) return &i->iomap; } +int iomap_iter_continue(const struct iomap_iter *iter, struct iomap *iomap, + struct iomap *srcmap, int ret); + +/** + * iomap_iter_next - finish the previous mapping and produce the next one + * @iter: iteration structure + * @iomap: mapping to finish and then repopulate + * @srcmap: source mapping to finish and then repopulate + * @begin: callback that produces a mapping for the current position + * @end: optional callback that finishes the previous mapping, or NULL + * + * Inline helper that implements the common body of an ->iomap_next() + * callback: it finishes the previous mapping via @end (if present), decides + * via iomap_iter_continue() whether to keep going, and obtains the next + * mapping via @begin. + * + * This helper is marked __always_inline so that when a caller passes + * compile-time-constant @begin and @end callbacks, the compiler can call them + * directly, avoiding the indirect-call overhead. + * + * Returns 1 to continue iterating, 0 once the range is fully consumed, or a + * negative errno on error. + */ +static __always_inline int iomap_iter_next(const struct iomap_iter *iter, + struct iomap *iomap, struct iomap *srcmap, + iomap_iter_begin_fn begin, iomap_iter_end_fn end) +{ + int ret = 0; + + if (iomap->length) { + if (end) { + /* + * Calculate how far the iter was advanced and the + * original length bytes for end(). + */ + ssize_t advanced = iter->pos - iter->iter_start_pos; + loff_t len; + + len = iomap_length_trim(iter, iter->iter_start_pos, + iter->len + advanced); + + ret = end(iter->inode, iter->iter_start_pos, len, + advanced, iter->flags, iomap); + } + ret = iomap_iter_continue(iter, iomap, srcmap, ret); + if (ret <= 0) + return ret; + } + + ret = begin(iter->inode, iter->pos, iter->len, iter->flags, iomap, + srcmap); + + return ret < 0 ? ret : 1; +} + +#define DEFINE_IOMAP_ITER_NEXT_END(name, begin_fn, end_fn) \ +int name(const struct iomap_iter *iter, struct iomap *iomap, \ + struct iomap *srcmap) \ +{ \ + return iomap_iter_next(iter, iomap, srcmap, begin_fn, end_fn); \ +} + +#define DEFINE_IOMAP_ITER_NEXT(name, begin_fn) \ + DEFINE_IOMAP_ITER_NEXT_END(name, begin_fn, NULL) + /* * Return the file offset for the first unchanged block after a short write. * diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index f78787e654f4c6..2c2fb39f049c3d 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h @@ -6,6 +6,7 @@ #define __LINUX_IRQCHIP_ARM_GIC_V5_H #include +#include #include #include @@ -324,6 +325,7 @@ struct gicv5_irs_chip_data { struct list_head entry; struct fwnode_handle *fwnode; void __iomem *irs_base; + struct resource res; u32 flags; u32 spi_min; u32 spi_range; diff --git a/include/linux/libnvdimm.h b/include/linux/libnvdimm.h index 28f086c4a1873c..d929d83abf3be1 100644 --- a/include/linux/libnvdimm.h +++ b/include/linux/libnvdimm.h @@ -126,6 +126,15 @@ struct nd_mapping_desc { struct bio; struct resource; struct nd_region; + +/* + * Provider flush callback return values: + * 0: flush completed synchronously + * <0: flush failed + * >0: flush completion was queued and @bio will be completed later + */ +#define NVDIMM_FLUSH_ASYNC 1 + struct nd_region_desc { struct resource *res; struct nd_mapping_desc *mapping; diff --git a/include/linux/liveupdate.h b/include/linux/liveupdate.h index 88722e5caf020b..63ea5417de849a 100644 --- a/include/linux/liveupdate.h +++ b/include/linux/liveupdate.h @@ -173,7 +173,9 @@ struct liveupdate_flb_ops { * @lock: A mutex that protects all fields within this structure, providing * the synchronization service for the FLB's ops. * @finished: True once the FLB's finish() callback has run. - * @retrieved: True once the FLB's retrieve() callback has run. + * @retrieve_status: Status code indicating whether retrieve() has been + * attempted. 0 means not attempted, 1 means successful, + * and negative value means it failed with that error code. */ struct luo_flb_private_state { refcount_t count; @@ -181,7 +183,7 @@ struct luo_flb_private_state { void *obj; struct mutex lock; bool finished; - bool retrieved; + int retrieve_status; }; /* @@ -243,6 +245,7 @@ int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp); void liveupdate_flb_put_incoming(struct liveupdate_flb *flb); int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp); +void liveupdate_flb_put_outgoing(struct liveupdate_flb *flb); #else /* CONFIG_LIVEUPDATE */ @@ -292,5 +295,9 @@ static inline int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, return -EOPNOTSUPP; } +static inline void liveupdate_flb_put_outgoing(struct liveupdate_flb *flb) +{ +} + #endif /* CONFIG_LIVEUPDATE */ #endif /* _LINUX_LIVEUPDATE_H */ diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index 4a5631906affb5..0d712e7be6ddc1 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -572,7 +572,7 @@ void maple_tree_init(void); void mas_destroy(struct ma_state *mas); void *mas_prev(struct ma_state *mas, unsigned long min); -void *mas_prev_range(struct ma_state *mas, unsigned long max); +void *mas_prev_range(struct ma_state *mas, unsigned long min); void *mas_next(struct ma_state *mas, unsigned long max); void *mas_next_range(struct ma_state *mas, unsigned long max); diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 8170bb8066a22f..32621fa690ac70 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -270,10 +270,15 @@ struct mem_cgroup { #endif int kmemcg_id; - struct memcg_vmstats_percpu __percpu *vmstats_percpu; - #ifdef CONFIG_CGROUP_WRITEBACK struct list_head cgwb_list; +#endif + + /* Keep the hot per-CPU stats pointer away from memory event counters. */ + struct memcg_vmstats_percpu __percpu *vmstats_percpu + ____cacheline_aligned_in_smp; + +#ifdef CONFIG_CGROUP_WRITEBACK struct wb_domain cgwb_domain; struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT]; #endif diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 7c9d66729c6095..06c58cb0577915 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -289,7 +289,7 @@ static inline void __remove_memory(u64 start, u64 size) {} /* Default online_type (MMOP_*) when new memory blocks are added. */ extern enum mmop mhp_get_default_online_type(void); extern void mhp_set_default_online_type(enum mmop online_type); -extern void __ref free_area_init_core_hotplug(struct pglist_data *pgdat); +int __ref free_area_init_core_hotplug(struct pglist_data *pgdat); extern int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory_resource(int nid, struct resource *resource, diff --git a/include/linux/mlx5/vport.h b/include/linux/mlx5/vport.h index ee34d3ed335f36..d5113ecb595882 100644 --- a/include/linux/mlx5/vport.h +++ b/include/linux/mlx5/vport.h @@ -61,7 +61,8 @@ u8 mlx5_query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport); int mlx5_modify_vport_admin_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport, u8 other_vport, u8 state); int mlx5_query_vport_max_tx_speed(struct mlx5_core_dev *mdev, u8 op_mod, - u16 vport, u8 other_vport, u32 *max_tx_speed); + u16 vport, u8 other_vport, + u32 *max_tx_speed, u8 *state); int mlx5_modify_vport_max_tx_speed(struct mlx5_core_dev *mdev, u8 opmod, u16 vport, u8 other_vport, u16 max_tx_speed); int mlx5_query_nic_vport_mac_address(struct mlx5_core_dev *mdev, diff --git a/include/linux/nfs4.h b/include/linux/nfs4.h index d87be1f25273a7..44e5e9fa12e1ee 100644 --- a/include/linux/nfs4.h +++ b/include/linux/nfs4.h @@ -171,133 +171,6 @@ Needs to be updated if more operations are defined in future.*/ #define LAST_NFS42_OP OP_REMOVEXATTR #define LAST_NFS4_OP LAST_NFS42_OP -enum nfsstat4 { - NFS4_OK = 0, - NFS4ERR_PERM = 1, - NFS4ERR_NOENT = 2, - NFS4ERR_IO = 5, - NFS4ERR_NXIO = 6, - NFS4ERR_ACCESS = 13, - NFS4ERR_EXIST = 17, - NFS4ERR_XDEV = 18, - /* Unused/reserved 19 */ - NFS4ERR_NOTDIR = 20, - NFS4ERR_ISDIR = 21, - NFS4ERR_INVAL = 22, - NFS4ERR_FBIG = 27, - NFS4ERR_NOSPC = 28, - NFS4ERR_ROFS = 30, - NFS4ERR_MLINK = 31, - NFS4ERR_NAMETOOLONG = 63, - NFS4ERR_NOTEMPTY = 66, - NFS4ERR_DQUOT = 69, - NFS4ERR_STALE = 70, - NFS4ERR_BADHANDLE = 10001, - NFS4ERR_BAD_COOKIE = 10003, - NFS4ERR_NOTSUPP = 10004, - NFS4ERR_TOOSMALL = 10005, - NFS4ERR_SERVERFAULT = 10006, - NFS4ERR_BADTYPE = 10007, - NFS4ERR_DELAY = 10008, - NFS4ERR_SAME = 10009, - NFS4ERR_DENIED = 10010, - NFS4ERR_EXPIRED = 10011, - NFS4ERR_LOCKED = 10012, - NFS4ERR_GRACE = 10013, - NFS4ERR_FHEXPIRED = 10014, - NFS4ERR_SHARE_DENIED = 10015, - NFS4ERR_WRONGSEC = 10016, - NFS4ERR_CLID_INUSE = 10017, - NFS4ERR_RESOURCE = 10018, - NFS4ERR_MOVED = 10019, - NFS4ERR_NOFILEHANDLE = 10020, - NFS4ERR_MINOR_VERS_MISMATCH = 10021, - NFS4ERR_STALE_CLIENTID = 10022, - NFS4ERR_STALE_STATEID = 10023, - NFS4ERR_OLD_STATEID = 10024, - NFS4ERR_BAD_STATEID = 10025, - NFS4ERR_BAD_SEQID = 10026, - NFS4ERR_NOT_SAME = 10027, - NFS4ERR_LOCK_RANGE = 10028, - NFS4ERR_SYMLINK = 10029, - NFS4ERR_RESTOREFH = 10030, - NFS4ERR_LEASE_MOVED = 10031, - NFS4ERR_ATTRNOTSUPP = 10032, - NFS4ERR_NO_GRACE = 10033, - NFS4ERR_RECLAIM_BAD = 10034, - NFS4ERR_RECLAIM_CONFLICT = 10035, - NFS4ERR_BADXDR = 10036, - NFS4ERR_LOCKS_HELD = 10037, - NFS4ERR_OPENMODE = 10038, - NFS4ERR_BADOWNER = 10039, - NFS4ERR_BADCHAR = 10040, - NFS4ERR_BADNAME = 10041, - NFS4ERR_BAD_RANGE = 10042, - NFS4ERR_LOCK_NOTSUPP = 10043, - NFS4ERR_OP_ILLEGAL = 10044, - NFS4ERR_DEADLOCK = 10045, - NFS4ERR_FILE_OPEN = 10046, - NFS4ERR_ADMIN_REVOKED = 10047, - NFS4ERR_CB_PATH_DOWN = 10048, - - /* nfs41 */ - NFS4ERR_BADIOMODE = 10049, - NFS4ERR_BADLAYOUT = 10050, - NFS4ERR_BAD_SESSION_DIGEST = 10051, - NFS4ERR_BADSESSION = 10052, - NFS4ERR_BADSLOT = 10053, - NFS4ERR_COMPLETE_ALREADY = 10054, - NFS4ERR_CONN_NOT_BOUND_TO_SESSION = 10055, - NFS4ERR_DELEG_ALREADY_WANTED = 10056, - NFS4ERR_BACK_CHAN_BUSY = 10057, /* backchan reqs outstanding */ - NFS4ERR_LAYOUTTRYLATER = 10058, - NFS4ERR_LAYOUTUNAVAILABLE = 10059, - NFS4ERR_NOMATCHING_LAYOUT = 10060, - NFS4ERR_RECALLCONFLICT = 10061, - NFS4ERR_UNKNOWN_LAYOUTTYPE = 10062, - NFS4ERR_SEQ_MISORDERED = 10063, /* unexpected seq.id in req */ - NFS4ERR_SEQUENCE_POS = 10064, /* [CB_]SEQ. op not 1st op */ - NFS4ERR_REQ_TOO_BIG = 10065, /* request too big */ - NFS4ERR_REP_TOO_BIG = 10066, /* reply too big */ - NFS4ERR_REP_TOO_BIG_TO_CACHE = 10067, /* rep. not all cached */ - NFS4ERR_RETRY_UNCACHED_REP = 10068, /* retry & rep. uncached */ - NFS4ERR_UNSAFE_COMPOUND = 10069, /* retry/recovery too hard */ - NFS4ERR_TOO_MANY_OPS = 10070, /* too many ops in [CB_]COMP */ - NFS4ERR_OP_NOT_IN_SESSION = 10071, /* op needs [CB_]SEQ. op */ - NFS4ERR_HASH_ALG_UNSUPP = 10072, /* hash alg. not supp. */ - /* Error 10073 is unused. */ - NFS4ERR_CLIENTID_BUSY = 10074, /* clientid has state */ - NFS4ERR_PNFS_IO_HOLE = 10075, /* IO to _SPARSE file hole */ - NFS4ERR_SEQ_FALSE_RETRY = 10076, /* retry not original */ - NFS4ERR_BAD_HIGH_SLOT = 10077, /* sequence arg bad */ - NFS4ERR_DEADSESSION = 10078, /* persistent session dead */ - NFS4ERR_ENCR_ALG_UNSUPP = 10079, /* SSV alg mismatch */ - NFS4ERR_PNFS_NO_LAYOUT = 10080, /* direct I/O with no layout */ - NFS4ERR_NOT_ONLY_OP = 10081, /* bad compound */ - NFS4ERR_WRONG_CRED = 10082, /* permissions:state change */ - NFS4ERR_WRONG_TYPE = 10083, /* current operation mismatch */ - NFS4ERR_DIRDELEG_UNAVAIL = 10084, /* no directory delegation */ - NFS4ERR_REJECT_DELEG = 10085, /* on callback */ - NFS4ERR_RETURNCONFLICT = 10086, /* outstanding layoutreturn */ - NFS4ERR_DELEG_REVOKED = 10087, /* deleg./layout revoked */ - - /* nfs42 */ - NFS4ERR_PARTNER_NOTSUPP = 10088, - NFS4ERR_PARTNER_NO_AUTH = 10089, - NFS4ERR_UNION_NOTSUPP = 10090, - NFS4ERR_OFFLOAD_DENIED = 10091, - NFS4ERR_WRONG_LFS = 10092, - NFS4ERR_BADLABEL = 10093, - NFS4ERR_OFFLOAD_NO_REQS = 10094, - - /* xattr (RFC8276) */ - NFS4ERR_NOXATTR = 10095, - NFS4ERR_XATTR2BIG = 10096, - - /* can be used for internal errors */ - NFS4ERR_FIRST_FREE -}; - /* error codes for internal client use */ #define NFS4ERR_RESET_TO_MDS 12001 #define NFS4ERR_RESET_TO_PNFS 12002 diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 11c5b31cfc7dae..e8144048cd0ecb 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -354,7 +354,7 @@ struct nfs4_layoutreturn { struct nfs4_xdr_opaque_data ld_private; }; -#define PNFS_LAYOUTSTATS_MAXSIZE 256 +#define PNFS_LAYOUTSTATS_MAXSIZE 384 struct nfs42_layoutstat_args; struct nfs42_layoutstat_devinfo; diff --git a/include/linux/pci.h b/include/linux/pci.h index 43f80d6189a7d9..66a4fda05793de 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -983,6 +983,7 @@ struct module; * function returns zero when the driver chooses to * take "ownership" of the device or an error code * (negative number) otherwise. + * The pci_device_id parameter is only valid during probe. * The probe function always gets called from process * context, so it can sleep. * @remove: The remove() function gets called whenever a device diff --git a/include/linux/remoteproc.h b/include/linux/remoteproc.h index 7c1546d4800825..de98462d588892 100644 --- a/include/linux/remoteproc.h +++ b/include/linux/remoteproc.h @@ -37,6 +37,7 @@ #include #include +#include #include #include #include @@ -145,7 +146,6 @@ struct rproc_ops { * a message. * @RPROC_RUNNING: device is up and running * @RPROC_CRASHED: device has crashed; need to start recovery - * @RPROC_DELETED: device is deleted * @RPROC_ATTACHED: device has been booted by another entity and the core * has attached to it * @RPROC_DETACHED: device has been booted by another entity and waiting @@ -163,10 +163,9 @@ enum rproc_state { RPROC_SUSPENDED = 1, RPROC_RUNNING = 2, RPROC_CRASHED = 3, - RPROC_DELETED = 4, - RPROC_ATTACHED = 5, - RPROC_DETACHED = 6, - RPROC_LAST = 7, + RPROC_ATTACHED = 4, + RPROC_DETACHED = 5, + RPROC_LAST = 6, }; /** @@ -259,7 +258,10 @@ enum rproc_features { * @subdevs: list of subdevices, to following the running state * @notifyids: idr for dynamically assigning rproc-wide unique notify ids * @index: index of this rproc device + * @attach_work: workqueue for attaching rproc * @crash_handler: workqueue for handling a crash + * @crash_handler_lock: serializes crash handler queueing and deletion + * @deleting: remoteproc deletion has begun * @crash_cnt: crash counter * @recovery_disabled: flag that state if recovery was disabled * @max_notifyid: largest allocated notify id. @@ -301,7 +303,10 @@ struct rproc { struct list_head subdevs; struct idr notifyids; int index; + struct work_struct attach_work; struct work_struct crash_handler; + spinlock_t crash_handler_lock; + bool deleting; unsigned int crash_cnt; bool recovery_disabled; int max_notifyid; diff --git a/include/linux/rsc_table.h b/include/linux/rsc_table.h index c6d6d553d8f11c..4cef11a2e3a2c0 100644 --- a/include/linux/rsc_table.h +++ b/include/linux/rsc_table.h @@ -337,17 +337,22 @@ static inline int rsc_table_for_each_entry(struct resource_table *table, int i, ret; for (i = 0; i < table->num; i++) { - int offset = table->offset[i]; - struct fw_rsc_hdr *hdr = (void *)table + offset; - int avail = table_sz - offset - sizeof(*hdr); - int rsc_offset = offset + sizeof(*hdr); - void *rsc = (void *)hdr + sizeof(*hdr); + u32 offset = table->offset[i]; + struct fw_rsc_hdr *hdr; + int avail, rsc_offset; + void *rsc; - if (avail < 0) { + if (offset < sizeof(*table) || offset >= table_sz || + table_sz - offset < sizeof(*hdr)) { dev_err(dev, "rsc table is truncated\n"); return -EINVAL; } + hdr = (void *)table + offset; + avail = table_sz - offset - sizeof(*hdr); + rsc_offset = offset + sizeof(*hdr); + rsc = (void *)hdr + sizeof(*hdr); + ret = cb(hdr->type, rsc, rsc_offset, avail, data); if (ret) return ret; diff --git a/include/linux/rtsx_usb.h b/include/linux/rtsx_usb.h index 276b509c03e364..0fc5a74700a8bb 100644 --- a/include/linux/rtsx_usb.h +++ b/include/linux/rtsx_usb.h @@ -61,6 +61,9 @@ struct rtsx_ucr { struct timer_list sg_timer; struct mutex dev_mutex; + + u16 card_status_cache; + bool card_status_valid; }; /* buffer size */ diff --git a/include/linux/soc/qcom/ubwc.h b/include/linux/soc/qcom/ubwc.h index 83d2c2a7116c00..f35c29440f4de6 100644 --- a/include/linux/soc/qcom/ubwc.h +++ b/include/linux/soc/qcom/ubwc.h @@ -8,6 +8,7 @@ #define __QCOM_UBWC_H__ #include +#include #include #include diff --git a/include/linux/sunrpc/xdrgen/_defs.h b/include/linux/sunrpc/xdrgen/_defs.h index 20c7270aa64d25..8f3776ef322958 100644 --- a/include/linux/sunrpc/xdrgen/_defs.h +++ b/include/linux/sunrpc/xdrgen/_defs.h @@ -25,6 +25,8 @@ typedef struct { #define XDR_void (0) #define XDR_bool (1) +#define XDR_short (1) +#define XDR_unsigned_short (1) #define XDR_int (1) #define XDR_unsigned_int (1) #define XDR_long (1) diff --git a/include/linux/sunrpc/xdrgen/nfs4_1.h b/include/linux/sunrpc/xdrgen/nfs4_1.h index 4ac54bdbd335b9..ed6df7690b5fb4 100644 --- a/include/linux/sunrpc/xdrgen/nfs4_1.h +++ b/include/linux/sunrpc/xdrgen/nfs4_1.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Generated by xdrgen. Manual edits will be lost. */ /* XDR specification file: ../../Documentation/sunrpc/xdr/nfs4_1.x */ -/* XDR specification modification time: Thu Jan 8 23:12:07 2026 */ +/* XDR specification modification time: Tue Jun 30 11:57:21 2026 */ #ifndef _LINUX_XDRGEN_NFS4_1_DEF_H #define _LINUX_XDRGEN_NFS4_1_DEF_H @@ -9,15 +9,149 @@ #include #include -typedef s64 int64_t; +typedef s32 int32_t; typedef u32 uint32_t; +typedef s64 int64_t; + +typedef u64 uint64_t; + +enum { NFS4_VERIFIER_SIZE = 8 }; + +enum { NFS4_FHSIZE = 128 }; + +enum nfsstat4 { + NFS4_OK = 0, + NFS4ERR_PERM = 1, + NFS4ERR_NOENT = 2, + NFS4ERR_IO = 5, + NFS4ERR_NXIO = 6, + NFS4ERR_ACCESS = 13, + NFS4ERR_EXIST = 17, + NFS4ERR_XDEV = 18, + NFS4ERR_NOTDIR = 20, + NFS4ERR_ISDIR = 21, + NFS4ERR_INVAL = 22, + NFS4ERR_FBIG = 27, + NFS4ERR_NOSPC = 28, + NFS4ERR_ROFS = 30, + NFS4ERR_MLINK = 31, + NFS4ERR_NAMETOOLONG = 63, + NFS4ERR_NOTEMPTY = 66, + NFS4ERR_DQUOT = 69, + NFS4ERR_STALE = 70, + NFS4ERR_BADHANDLE = 10001, + NFS4ERR_BAD_COOKIE = 10003, + NFS4ERR_NOTSUPP = 10004, + NFS4ERR_TOOSMALL = 10005, + NFS4ERR_SERVERFAULT = 10006, + NFS4ERR_BADTYPE = 10007, + NFS4ERR_DELAY = 10008, + NFS4ERR_SAME = 10009, + NFS4ERR_DENIED = 10010, + NFS4ERR_EXPIRED = 10011, + NFS4ERR_LOCKED = 10012, + NFS4ERR_GRACE = 10013, + NFS4ERR_FHEXPIRED = 10014, + NFS4ERR_SHARE_DENIED = 10015, + NFS4ERR_WRONGSEC = 10016, + NFS4ERR_CLID_INUSE = 10017, + NFS4ERR_RESOURCE = 10018, + NFS4ERR_MOVED = 10019, + NFS4ERR_NOFILEHANDLE = 10020, + NFS4ERR_MINOR_VERS_MISMATCH = 10021, + NFS4ERR_STALE_CLIENTID = 10022, + NFS4ERR_STALE_STATEID = 10023, + NFS4ERR_OLD_STATEID = 10024, + NFS4ERR_BAD_STATEID = 10025, + NFS4ERR_BAD_SEQID = 10026, + NFS4ERR_NOT_SAME = 10027, + NFS4ERR_LOCK_RANGE = 10028, + NFS4ERR_SYMLINK = 10029, + NFS4ERR_RESTOREFH = 10030, + NFS4ERR_LEASE_MOVED = 10031, + NFS4ERR_ATTRNOTSUPP = 10032, + NFS4ERR_NO_GRACE = 10033, + NFS4ERR_RECLAIM_BAD = 10034, + NFS4ERR_RECLAIM_CONFLICT = 10035, + NFS4ERR_BADXDR = 10036, + NFS4ERR_LOCKS_HELD = 10037, + NFS4ERR_OPENMODE = 10038, + NFS4ERR_BADOWNER = 10039, + NFS4ERR_BADCHAR = 10040, + NFS4ERR_BADNAME = 10041, + NFS4ERR_BAD_RANGE = 10042, + NFS4ERR_LOCK_NOTSUPP = 10043, + NFS4ERR_OP_ILLEGAL = 10044, + NFS4ERR_DEADLOCK = 10045, + NFS4ERR_FILE_OPEN = 10046, + NFS4ERR_ADMIN_REVOKED = 10047, + NFS4ERR_CB_PATH_DOWN = 10048, + NFS4ERR_BADIOMODE = 10049, + NFS4ERR_BADLAYOUT = 10050, + NFS4ERR_BAD_SESSION_DIGEST = 10051, + NFS4ERR_BADSESSION = 10052, + NFS4ERR_BADSLOT = 10053, + NFS4ERR_COMPLETE_ALREADY = 10054, + NFS4ERR_CONN_NOT_BOUND_TO_SESSION = 10055, + NFS4ERR_DELEG_ALREADY_WANTED = 10056, + NFS4ERR_BACK_CHAN_BUSY = 10057, + NFS4ERR_LAYOUTTRYLATER = 10058, + NFS4ERR_LAYOUTUNAVAILABLE = 10059, + NFS4ERR_NOMATCHING_LAYOUT = 10060, + NFS4ERR_RECALLCONFLICT = 10061, + NFS4ERR_UNKNOWN_LAYOUTTYPE = 10062, + NFS4ERR_SEQ_MISORDERED = 10063, + NFS4ERR_SEQUENCE_POS = 10064, + NFS4ERR_REQ_TOO_BIG = 10065, + NFS4ERR_REP_TOO_BIG = 10066, + NFS4ERR_REP_TOO_BIG_TO_CACHE = 10067, + NFS4ERR_RETRY_UNCACHED_REP = 10068, + NFS4ERR_UNSAFE_COMPOUND = 10069, + NFS4ERR_TOO_MANY_OPS = 10070, + NFS4ERR_OP_NOT_IN_SESSION = 10071, + NFS4ERR_HASH_ALG_UNSUPP = 10072, + NFS4ERR_CLIENTID_BUSY = 10074, + NFS4ERR_PNFS_IO_HOLE = 10075, + NFS4ERR_SEQ_FALSE_RETRY = 10076, + NFS4ERR_BAD_HIGH_SLOT = 10077, + NFS4ERR_DEADSESSION = 10078, + NFS4ERR_ENCR_ALG_UNSUPP = 10079, + NFS4ERR_PNFS_NO_LAYOUT = 10080, + NFS4ERR_NOT_ONLY_OP = 10081, + NFS4ERR_WRONG_CRED = 10082, + NFS4ERR_WRONG_TYPE = 10083, + NFS4ERR_DIRDELEG_UNAVAIL = 10084, + NFS4ERR_REJECT_DELEG = 10085, + NFS4ERR_RETURNCONFLICT = 10086, + NFS4ERR_DELEG_REVOKED = 10087, + NFS4ERR_PARTNER_NOTSUPP = 10088, + NFS4ERR_PARTNER_NO_AUTH = 10089, + NFS4ERR_UNION_NOTSUPP = 10090, + NFS4ERR_OFFLOAD_DENIED = 10091, + NFS4ERR_WRONG_LFS = 10092, + NFS4ERR_BADLABEL = 10093, + NFS4ERR_OFFLOAD_NO_REQS = 10094, + NFS4ERR_NOXATTR = 10095, + NFS4ERR_XATTR2BIG = 10096, +}; + +typedef enum nfsstat4 nfsstat4; + +typedef opaque attrlist4; + typedef struct { u32 count; uint32_t *element; } bitmap4; +typedef u8 verifier4[NFS4_VERIFIER_SIZE]; + +typedef uint64_t nfs_cookie4; + +typedef opaque nfs_fh4; + typedef opaque utf8string; typedef utf8string utf8str_cis; @@ -26,11 +160,30 @@ typedef utf8string utf8str_cs; typedef utf8string utf8str_mixed; +typedef utf8str_cs component4; + +typedef utf8str_cs linktext4; + +typedef struct { + u32 count; + component4 *element; +} pathname4; + struct nfstime4 { int64_t seconds; uint32_t nseconds; }; +struct fattr4 { + bitmap4 attrmask; + attrlist4 attr_vals; +}; + +struct stateid4 { + uint32_t seqid; + u8 other[12]; +}; + typedef bool fattr4_offline; enum { FATTR4_OFFLINE = 83 }; @@ -216,11 +369,98 @@ enum { FATTR4_POSIX_DEFAULT_ACL = 91 }; enum { FATTR4_POSIX_ACCESS_ACL = 92 }; -#define NFS4_int64_t_sz \ - (XDR_hyper) +enum notify_type4 { + NOTIFY4_CHANGE_CHILD_ATTRS = 0, + NOTIFY4_CHANGE_DIR_ATTRS = 1, + NOTIFY4_REMOVE_ENTRY = 2, + NOTIFY4_ADD_ENTRY = 3, + NOTIFY4_RENAME_ENTRY = 4, + NOTIFY4_CHANGE_COOKIE_VERIFIER = 5, +}; + +typedef enum notify_type4 notify_type4; + +struct notify_entry4 { + component4 ne_file; + struct fattr4 ne_attrs; +}; + +struct prev_entry4 { + struct notify_entry4 pe_prev_entry; + nfs_cookie4 pe_prev_entry_cookie; +}; + +struct notify_remove4 { + struct notify_entry4 nrm_old_entry; + nfs_cookie4 nrm_old_entry_cookie; +}; + +struct notify_add4 { + struct { + u32 count; + struct notify_remove4 *element; + } nad_old_entry; + struct notify_entry4 nad_new_entry; + struct { + u32 count; + nfs_cookie4 *element; + } nad_new_entry_cookie; + struct { + u32 count; + struct prev_entry4 *element; + } nad_prev_entry; + bool nad_last_entry; +}; + +struct notify_attr4 { + struct notify_entry4 na_changed_entry; +}; + +struct notify_rename4 { + struct notify_remove4 nrn_old_entry; + struct notify_add4 nrn_new_entry; +}; + +struct notify_verifier4 { + verifier4 nv_old_cookieverf; + verifier4 nv_new_cookieverf; +}; + +typedef opaque notifylist4; + +struct notify4 { + bitmap4 notify_mask; + notifylist4 notify_vals; +}; + +struct CB_NOTIFY4args { + struct stateid4 cna_stateid; + nfs_fh4 cna_fh; + struct { + u32 count; + struct notify4 *element; + } cna_changes; +}; + +struct CB_NOTIFY4res { + nfsstat4 cnr_status; +}; + +#define NFS4_int32_t_sz \ + (XDR_int) #define NFS4_uint32_t_sz \ (XDR_unsigned_int) +#define NFS4_int64_t_sz \ + (XDR_hyper) +#define NFS4_uint64_t_sz \ + (XDR_unsigned_hyper) +#define NFS4_nfsstat4_sz (XDR_int) +#define NFS4_attrlist4_sz (XDR_unsigned_int) #define NFS4_bitmap4_sz (XDR_unsigned_int) +#define NFS4_verifier4_sz (XDR_QUADLEN(NFS4_VERIFIER_SIZE)) +#define NFS4_nfs_cookie4_sz \ + (NFS4_uint64_t_sz) +#define NFS4_nfs_fh4_sz (XDR_unsigned_int + XDR_QUADLEN(NFS4_FHSIZE)) #define NFS4_utf8string_sz (XDR_unsigned_int) #define NFS4_utf8str_cis_sz \ (NFS4_utf8string_sz) @@ -228,8 +468,17 @@ enum { FATTR4_POSIX_ACCESS_ACL = 92 }; (NFS4_utf8string_sz) #define NFS4_utf8str_mixed_sz \ (NFS4_utf8string_sz) +#define NFS4_component4_sz \ + (NFS4_utf8str_cs_sz) +#define NFS4_linktext4_sz \ + (NFS4_utf8str_cs_sz) +#define NFS4_pathname4_sz (XDR_unsigned_int) #define NFS4_nfstime4_sz \ (NFS4_int64_t_sz + NFS4_uint32_t_sz) +#define NFS4_fattr4_sz \ + (NFS4_bitmap4_sz + NFS4_attrlist4_sz) +#define NFS4_stateid4_sz \ + (NFS4_uint32_t_sz + XDR_QUADLEN(12)) #define NFS4_fattr4_offline_sz \ (XDR_bool) #define NFS4_open_arguments4_sz \ @@ -259,5 +508,27 @@ enum { FATTR4_POSIX_ACCESS_ACL = 92 }; (NFS4_aclscope4_sz) #define NFS4_fattr4_posix_default_acl_sz (XDR_unsigned_int) #define NFS4_fattr4_posix_access_acl_sz (XDR_unsigned_int) +#define NFS4_notify_type4_sz (XDR_int) +#define NFS4_notify_entry4_sz \ + (NFS4_component4_sz + NFS4_fattr4_sz) +#define NFS4_prev_entry4_sz \ + (NFS4_notify_entry4_sz + NFS4_nfs_cookie4_sz) +#define NFS4_notify_remove4_sz \ + (NFS4_notify_entry4_sz + NFS4_nfs_cookie4_sz) +#define NFS4_notify_add4_sz \ + (XDR_unsigned_int + (1 * (NFS4_notify_remove4_sz)) + NFS4_notify_entry4_sz + XDR_unsigned_int + (1 * (NFS4_nfs_cookie4_sz)) + XDR_unsigned_int + (1 * (NFS4_prev_entry4_sz)) + XDR_bool) +#define NFS4_notify_attr4_sz \ + (NFS4_notify_entry4_sz) +#define NFS4_notify_rename4_sz \ + (NFS4_notify_remove4_sz + NFS4_notify_add4_sz) +#define NFS4_notify_verifier4_sz \ + (NFS4_verifier4_sz + NFS4_verifier4_sz) +#define NFS4_notifylist4_sz (XDR_unsigned_int) +#define NFS4_notify4_sz \ + (NFS4_bitmap4_sz + NFS4_notifylist4_sz) +#define NFS4_CB_NOTIFY4args_sz \ + (NFS4_stateid4_sz + NFS4_nfs_fh4_sz + XDR_unsigned_int) +#define NFS4_CB_NOTIFY4res_sz \ + (NFS4_nfsstat4_sz) #endif /* _LINUX_XDRGEN_NFS4_1_DEF_H */ diff --git a/include/linux/sunrpc/xdrgen/nlm3.h b/include/linux/sunrpc/xdrgen/nlm3.h index 897e7d91807c62..0fc627031d8a93 100644 --- a/include/linux/sunrpc/xdrgen/nlm3.h +++ b/include/linux/sunrpc/xdrgen/nlm3.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Generated by xdrgen. Manual edits will be lost. */ /* XDR specification file: ../../Documentation/sunrpc/xdr/nlm3.x */ -/* XDR specification modification time: Thu Apr 23 10:56:34 2026 */ +/* XDR specification modification time: Mon Jun 29 20:42:29 2026 */ #ifndef _LINUX_XDRGEN_NLM3_DEF_H #define _LINUX_XDRGEN_NLM3_DEF_H diff --git a/include/linux/sunrpc/xdrgen/nlm4.h b/include/linux/sunrpc/xdrgen/nlm4.h index e95e8f105624d8..77860a3d1c1e94 100644 --- a/include/linux/sunrpc/xdrgen/nlm4.h +++ b/include/linux/sunrpc/xdrgen/nlm4.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Generated by xdrgen. Manual edits will be lost. */ /* XDR specification file: ../../Documentation/sunrpc/xdr/nlm4.x */ -/* XDR specification modification time: Thu Dec 25 13:10:19 2025 */ +/* XDR specification modification time: Mon Jun 29 20:42:29 2026 */ #ifndef _LINUX_XDRGEN_NLM4_DEF_H #define _LINUX_XDRGEN_NLM4_DEF_H diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h index f548fea2adec8f..18be159bbc341d 100644 --- a/include/linux/uprobes.h +++ b/include/linux/uprobes.h @@ -186,9 +186,6 @@ struct xol_area; struct uprobes_state { struct xol_area *xol_area; -#ifdef CONFIG_X86_64 - struct hlist_head head_tramps; -#endif }; typedef int (*uprobe_write_verify_t)(struct page *page, unsigned long vaddr, @@ -238,8 +235,6 @@ extern void uprobe_handle_trampoline(struct pt_regs *regs); extern void *arch_uretprobe_trampoline(unsigned long *psize); extern unsigned long uprobe_get_trampoline_vaddr(void); extern void uprobe_copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len); -extern void arch_uprobe_clear_state(struct mm_struct *mm); -extern void arch_uprobe_init_state(struct mm_struct *mm); extern void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr); extern void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr); extern unsigned long arch_uprobe_get_xol_area(void); diff --git a/include/linux/usb.h b/include/linux/usb.h index 1da4ad1610bca0..49ab8dbb885f65 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -1185,7 +1185,8 @@ extern ssize_t usb_show_dynids(struct usb_dynids *dynids, char *buf); * interface. It may also use usb_set_interface() to specify the * appropriate altsetting. If unwilling to manage the interface, * return -ENODEV, if genuine IO errors occurred, an appropriate - * negative errno value. + * negative errno value. The usb_device_id parameter is only valid during + * probe. * @disconnect: Called when the interface is no longer accessible, usually * because its device has been (or is being) disconnected or the * driver module is being unloaded. diff --git a/include/linux/usb/pd.h b/include/linux/usb/pd.h index 337a5485af7c7c..ee360dedeaa65d 100644 --- a/include/linux/usb/pd.h +++ b/include/linux/usb/pd.h @@ -493,7 +493,7 @@ static inline unsigned int pdo_epr_avs_apdo_min_voltage_mv(u32 pdo) static inline unsigned int pdo_epr_avs_apdo_max_voltage_mv(u32 pdo) { - return FIELD_GET(PDO_EPR_AVS_APDO_MIN_VOLT, pdo) * 100; + return FIELD_GET(PDO_EPR_AVS_APDO_MAX_VOLT, pdo) * 100; } static inline unsigned int pdo_epr_avs_apdo_pdp_w(u32 pdo) diff --git a/include/linux/virtio.h b/include/linux/virtio.h index 93e573c565635a..f923e42cfd0110 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -213,6 +213,7 @@ int virtio_device_freeze(struct virtio_device *dev); int virtio_device_restore(struct virtio_device *dev); #endif void virtio_reset_device(struct virtio_device *dev); +void virtio_device_shutdown(struct virtio_device *dev); int virtio_device_reset_prepare(struct virtio_device *dev); int virtio_device_reset_done(struct virtio_device *dev); diff --git a/include/linux/virtio_net.h b/include/linux/virtio_net.h index f36d21b5bc19e2..c381b916c1b54a 100644 --- a/include/linux/virtio_net.h +++ b/include/linux/virtio_net.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -179,6 +180,9 @@ retry: if (skb->ip_summed == CHECKSUM_PARTIAL && skb->csum_offset != offsetof(struct tcphdr, check)) return -EINVAL; + + BUILD_BUG_ON(TCP_MIN_GSO_SIZE * GSO_MAX_SEGS < GSO_MAX_SIZE); + gso_size = max(gso_size, TCP_MIN_GSO_SIZE); break; } diff --git a/include/media/rc-map.h b/include/media/rc-map.h index d90e4611b06644..d95ed3e96de283 100644 --- a/include/media/rc-map.h +++ b/include/media/rc-map.h @@ -262,7 +262,6 @@ struct rc_map *rc_map_get(const char *name); #define RC_MAP_GENIUS_TVGO_A11MCE "rc-genius-tvgo-a11mce" #define RC_MAP_GOTVIEW7135 "rc-gotview7135" #define RC_MAP_HAUPPAUGE "rc-hauppauge" -#define RC_MAP_HAUPPAUGE_NEW "rc-hauppauge" #define RC_MAP_HISI_POPLAR "rc-hisi-poplar" #define RC_MAP_HISI_TV_DEMO "rc-hisi-tv-demo" #define RC_MAP_IMON_MCE "rc-imon-mce" @@ -309,7 +308,6 @@ struct rc_map *rc_map_get(const char *name); #define RC_MAP_PROTEUS_2309 "rc-proteus-2309" #define RC_MAP_PURPLETV "rc-purpletv" #define RC_MAP_PV951 "rc-pv951" -#define RC_MAP_RC5_TV "rc-rc5-tv" #define RC_MAP_RC6_MCE "rc-rc6-mce" #define RC_MAP_REAL_AUDIO_220_32_KEYS "rc-real-audio-220-32-keys" #define RC_MAP_REDDO "rc-reddo" diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index 3df59849dcbeaa..8115435b5d2af2 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -919,9 +919,9 @@ static inline void hci_discovery_filter_clear(struct hci_dev *hdev) hdev->discovery.result_filtering = false; hdev->discovery.report_invalid_rssi = true; hdev->discovery.rssi = HCI_RSSI_INVALID; - hdev->discovery.uuid_count = 0; spin_lock(&hdev->discovery.lock); + hdev->discovery.uuid_count = 0; kfree(hdev->discovery.uuids); hdev->discovery.uuids = NULL; spin_unlock(&hdev->discovery.lock); diff --git a/include/net/bluetooth/l2cap.h b/include/net/bluetooth/l2cap.h index ef6ce1c20a4f05..3d9a32094347cd 100644 --- a/include/net/bluetooth/l2cap.h +++ b/include/net/bluetooth/l2cap.h @@ -699,7 +699,12 @@ struct l2cap_rx_busy { struct l2cap_pinfo { struct bt_sock bt; + + /* With owning sk_socket chan may be read without lock, other access + * should hold lock_sock. + */ struct l2cap_chan *chan; + struct list_head rx_busy; }; diff --git a/include/net/inetpeer.h b/include/net/inetpeer.h index f475757daafba9..414e9adf4c5114 100644 --- a/include/net/inetpeer.h +++ b/include/net/inetpeer.h @@ -35,6 +35,7 @@ struct inetpeer_addr { struct inet_peer { struct rb_node rb_node; + u64 hash; struct inetpeer_addr daddr; u32 metrics[RTAX_MAX]; @@ -125,6 +126,9 @@ static inline int inetpeer_addr_cmp(const struct inetpeer_addr *a, { int i, n; + if (a->family != b->family) + return a->family < b->family ? -1 : 1; + if (a->family == AF_INET) n = sizeof(a->a4) / sizeof(u32); else diff --git a/include/net/libeth/xsk.h b/include/net/libeth/xsk.h index 5dcc0d7f65b763..a452b7828ce4ec 100644 --- a/include/net/libeth/xsk.h +++ b/include/net/libeth/xsk.h @@ -196,7 +196,7 @@ __libeth_xsk_xmit_fill_buf_md(const struct xdp_desc *xdesc, struct libeth_xdp_tx_desc desc; struct xdp_desc_ctx ctx; - ctx = xsk_buff_raw_get_ctx(sq->pool, xdesc->addr); + ctx = xsk_buff_raw_get_ctx(sq->pool, xdesc->addr, xdesc->options); desc = (typeof(desc)){ .addr = ctx.dma, __libeth_xdp_tx_len(xdesc->len), diff --git a/include/net/netfilter/nf_conntrack_helper.h b/include/net/netfilter/nf_conntrack_helper.h index c761cd8158b253..309f74ea4d0986 100644 --- a/include/net/netfilter/nf_conntrack_helper.h +++ b/include/net/netfilter/nf_conntrack_helper.h @@ -108,6 +108,7 @@ void nf_ct_helper_init(struct nf_conntrack_helper *helper, int nf_conntrack_helper_register(struct nf_conntrack_helper *, struct nf_conntrack_helper **); int __nf_conntrack_helper_register(struct nf_conntrack_helper *); void nf_conntrack_helper_unregister(struct nf_conntrack_helper *); +void nf_conntrack_helper_release(struct nf_conntrack_helper *); int nf_conntrack_helpers_register(struct nf_conntrack_helper *, unsigned int, struct nf_conntrack_helper **); diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h index 3be612145c130c..85de3473da240f 100644 --- a/include/net/netfilter/nf_tables.h +++ b/include/net/netfilter/nf_tables.h @@ -870,8 +870,6 @@ struct nft_elem_priv *nft_set_elem_init(const struct nft_set *set, const u32 *key, const u32 *key_end, const u32 *data, u64 timeout, u64 expiration, gfp_t gfp); -int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set, - struct nft_expr *expr_array[]); void nft_set_elem_expr_destroy(const struct nft_ctx *ctx, struct nft_set_elem_expr *elem_expr); void nft_set_elem_destroy(const struct nft_set *set, diff --git a/include/net/pkt_sched.h b/include/net/pkt_sched.h index 18a419cd9d949b..90d3e7943b19ab 100644 --- a/include/net/pkt_sched.h +++ b/include/net/pkt_sched.h @@ -12,6 +12,7 @@ #define DEFAULT_TX_QUEUE_LEN 1000 #define STAB_SIZE_LOG_MAX 30 +#define QDISC_PKT_LEN_MAX (1 << 20) /* 1 MiB */ struct qdisc_walker { int stop; diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index b344789f5df836..d94aeb506379c4 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -240,6 +240,7 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) * xsk_buff_raw_get_ctx - get &xdp_desc context * @pool: XSk buff pool desc address belongs to * @addr: desc address (from userspace) + * @options: desc options (from userspace) * * Wrapper for xp_raw_get_ctx() to be used in drivers, see its kdoc for * details. @@ -248,9 +249,9 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) * pointer, if it is present (initialized to %NULL otherwise). */ static inline struct xdp_desc_ctx -xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) +xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr, u32 options) { - return xp_raw_get_ctx(pool, addr); + return xp_raw_get_ctx(pool, addr, options); } #define XDP_TXMD_FLAGS_VALID ( \ @@ -318,18 +319,20 @@ xsk_tx_metadata_request(const struct xsk_buff_pool *pool, } static inline struct xsk_tx_metadata * -__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data) +__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data, + unsigned int options) { - if (!pool->tx_metadata_len) + if (!pool->tx_metadata_len || !(options & XDP_TX_METADATA)) return NULL; return data - pool->tx_metadata_len; } static inline struct xsk_tx_metadata * -xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr) +xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr, u32 options) { - return __xsk_buff_get_metadata(pool, xp_raw_get_data(pool, addr)); + return __xsk_buff_get_metadata(pool, xp_raw_get_data(pool, addr), + options); } static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp) @@ -510,7 +513,7 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) } static inline struct xdp_desc_ctx -xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) +xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr, u32 options) { return (struct xdp_desc_ctx){ }; } @@ -530,13 +533,14 @@ xsk_tx_metadata_request(const struct xsk_buff_pool *pool, } static inline struct xsk_tx_metadata * -__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data) +__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data, + unsigned int options) { return NULL; } static inline struct xsk_tx_metadata * -xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr) +xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr, u32 options) { return NULL; } diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h index 2bb1d122b1bc6b..a7df573784fd6a 100644 --- a/include/net/xsk_buff_pool.h +++ b/include/net/xsk_buff_pool.h @@ -154,7 +154,8 @@ struct xdp_desc_ctx { struct xsk_tx_metadata *meta; }; -struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr); +struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr, + u32 options); static inline dma_addr_t xp_get_dma(struct xdp_buff_xsk *xskb) { diff --git a/include/rdma/uverbs_ioctl.h b/include/rdma/uverbs_ioctl.h index 24fd362130233f..80f3ba6663d06e 100644 --- a/include/rdma/uverbs_ioctl.h +++ b/include/rdma/uverbs_ioctl.h @@ -1151,4 +1151,24 @@ static inline int ib_respond_empty_udata(struct ib_udata *udata) return 0; } +/** + * ib_no_udata_io - Ensure no input data and zero fill the response buffer + * @udata: The system call's ib_udata struct + * + * Driver ops which do not accept any input data and do not provide any response + * data may call this at the beginning of their handler to fully adhere to the + * uAPI forward/backward compatibility rules. + * + * Return: Negative failure code if the op should be denied, 0 otherwise. + */ +static inline int ib_no_udata_io(struct ib_udata *udata) +{ + int ret = ib_is_udata_in_empty(udata); + + if (ret) + return ret; + + return ib_respond_empty_udata(udata); +} + #endif diff --git a/include/sound/sdca_fdl.h b/include/sound/sdca_fdl.h index dc33927b82bde6..bc3600cdd2ec84 100644 --- a/include/sound/sdca_fdl.h +++ b/include/sound/sdca_fdl.h @@ -83,6 +83,10 @@ static inline int sdca_fdl_alloc_state(struct sdca_interrupt *interrupt) return 0; } +static inline void sdca_fdl_free_state(struct sdca_interrupt *interrupt) +{ +} + static inline int sdca_fdl_process(struct sdca_interrupt *interrupt) { return 0; diff --git a/include/uapi/drm/panthor_drm.h b/include/uapi/drm/panthor_drm.h index 0e455d91e77d44..f857f4530eb2a3 100644 --- a/include/uapi/drm/panthor_drm.h +++ b/include/uapi/drm/panthor_drm.h @@ -601,6 +601,18 @@ enum drm_panthor_vm_bind_op_flags { */ DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED = 1 << 2, + /** + * @DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE: Sparsely map a virtual memory range + * + * Only valid with DRM_PANTHOR_VM_BIND_OP_TYPE_MAP. + * + * When this flag is set, the whole vm_bind range is mapped over a dummy object in a cyclic + * fashion, and all GPU reads from addresses in the range return undefined values. This flag + * being set means drm_panthor_vm_bind_op::bo_offset and drm_panthor_vm_bind_op::bo_handle + * must both be set to 0. DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC must also be set. + */ + DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE = 1 << 3, + /** * @DRM_PANTHOR_VM_BIND_OP_TYPE_MASK: Mask used to determine the type of operation. */ diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 48e9f1fdb78dde..50c80af4ad4ef8 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -2589,6 +2589,12 @@ enum drm_xe_ras_error_component { DRM_XE_RAS_ERR_COMP_CORE_COMPUTE = 1, /** @DRM_XE_RAS_ERR_COMP_SOC_INTERNAL: SoC Internal Error */ DRM_XE_RAS_ERR_COMP_SOC_INTERNAL, + /** @DRM_XE_RAS_ERR_COMP_DEVICE_MEMORY: Device Memory Error */ + DRM_XE_RAS_ERR_COMP_DEVICE_MEMORY, + /** @DRM_XE_RAS_ERR_COMP_PCIE: PCIe Subsystem Error */ + DRM_XE_RAS_ERR_COMP_PCIE, + /** @DRM_XE_RAS_ERR_COMP_FABRIC: Fabric Subsystem Error */ + DRM_XE_RAS_ERR_COMP_FABRIC, /** @DRM_XE_RAS_ERR_COMP_MAX: Max Error */ DRM_XE_RAS_ERR_COMP_MAX /* non-ABI */ }; @@ -2606,7 +2612,10 @@ enum drm_xe_ras_error_component { */ #define DRM_XE_RAS_ERROR_COMPONENT_NAMES { \ [DRM_XE_RAS_ERR_COMP_CORE_COMPUTE] = "core-compute", \ - [DRM_XE_RAS_ERR_COMP_SOC_INTERNAL] = "soc-internal" \ + [DRM_XE_RAS_ERR_COMP_SOC_INTERNAL] = "soc-internal", \ + [DRM_XE_RAS_ERR_COMP_DEVICE_MEMORY] = "device-memory", \ + [DRM_XE_RAS_ERR_COMP_PCIE] = "pcie", \ + [DRM_XE_RAS_ERR_COMP_FABRIC] = "fabric", \ } #if defined(__cplusplus) diff --git a/include/uapi/linux/if_xdp.h b/include/uapi/linux/if_xdp.h index 23a0627814687e..50d67df78911f5 100644 --- a/include/uapi/linux/if_xdp.h +++ b/include/uapi/linux/if_xdp.h @@ -149,6 +149,7 @@ struct xsk_tx_metadata { __u16 csum_start; /* Offset from csum_start where checksum should be stored. */ __u16 csum_offset; + __u32 reserved; /* XDP_TXMD_FLAGS_LAUNCH_TIME */ /* Launch time in nanosecond against the PTP HW Clock */ diff --git a/include/uapi/linux/nfs4.h b/include/uapi/linux/nfs4.h index 4273e0249fcbb5..289205b53a0858 100644 --- a/include/uapi/linux/nfs4.h +++ b/include/uapi/linux/nfs4.h @@ -17,11 +17,9 @@ #include #define NFS4_BITMAP_SIZE 3 -#define NFS4_VERIFIER_SIZE 8 #define NFS4_STATEID_SEQID_SIZE 4 #define NFS4_STATEID_OTHER_SIZE 12 #define NFS4_STATEID_SIZE (NFS4_STATEID_SEQID_SIZE + NFS4_STATEID_OTHER_SIZE) -#define NFS4_FHSIZE 128 #define NFS4_MAXPATHLEN PATH_MAX #define NFS4_MAXNAMLEN NAME_MAX #define NFS4_OPAQUE_LIMIT 1024 diff --git a/include/uapi/linux/time_types.h b/include/uapi/linux/time_types.h index bcc0002115d399..03a0d8aaadca59 100644 --- a/include/uapi/linux/time_types.h +++ b/include/uapi/linux/time_types.h @@ -30,7 +30,7 @@ struct __kernel_old_timeval { struct __kernel_old_timespec { __kernel_old_time_t tv_sec; /* seconds */ - long tv_nsec; /* nanoseconds */ + __kernel_long_t tv_nsec; /* nanoseconds */ }; struct __kernel_old_itimerval { diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index 88c594c6d7fcd5..f95859e4e3f090 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -169,6 +169,7 @@ config SCHED_CORE config SCHED_CLASS_EXT bool "Extensible Scheduling Class" depends on BPF_SYSCALL && BPF_JIT && DEBUG_INFO_BTF + select GENERIC_ALLOCATOR select STACKTRACE if STACKTRACE_SUPPORT help This option enables a new scheduler class sched_ext (SCX), which diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 80b7b8a6944645..529c0f6d7d0a80 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -484,8 +484,12 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) kaddr = kbase + (u32)(vmf->address); if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) - /* Make a reasonable effort to address impossible case */ - return VM_FAULT_RETRY; + /* + * A failed lock means a possible deadlock was detected. Don't + * return VM_FAULT_RETRY: this handler never took mmap_lock, but + * the fault path would re-take it on retry and deadlock. Fail. + */ + return VM_FAULT_SIGBUS; page = vmalloc_to_page((void *)kaddr); if (page) { @@ -853,6 +857,8 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, uaddr &= PAGE_MASK; kaddr = bpf_arena_get_kern_vm_start(arena) + uaddr; full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; + if (full_uaddr < arena->user_vm_start) + return; uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT)); if (full_uaddr >= uaddr_end) return; diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 1433809bb166a3..3983b4ce73c811 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -295,7 +295,6 @@ BTF_ID(func, bpf_lsm_bpf_map_create) BTF_ID(func, bpf_lsm_bpf_map_free) BTF_ID(func, bpf_lsm_bpf_prog) BTF_ID(func, bpf_lsm_bpf_prog_load) -BTF_ID(func, bpf_lsm_bpf_prog_free) BTF_ID(func, bpf_lsm_bpf_token_create) BTF_ID(func, bpf_lsm_bpf_token_free) BTF_ID(func, bpf_lsm_bpf_token_cmd) diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 51b16e5f5534e0..4e7a48c02be5cd 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -445,6 +445,18 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, goto errout; } + /* + * A >8 byte return value is passed back in a register pair, + * which the struct_ops trampoline does not preserve (only + * 8 bytes of the return value are saved and restored). + */ + if (st_ops->func_models[i].ret_size > 8) { + pr_warn("func ptr %s in struct %s has a >8 byte return value, which is not supported\n", + mname, st_ops->name); + err = -EOPNOTSUPP; + goto errout; + } + stub_func_addr = *(void **)(st_ops->cfi_stubs + moff); err = prepare_arg_info(btf, st_ops->name, mname, func_proto, stub_func_addr, diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index c4673a54c4baf2..608be952717d41 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -3669,7 +3669,7 @@ static int btf_get_field_type(const struct btf *btf, const struct btf_type *var_ { BPF_LIST_NODE, "bpf_list_node", false }, { BPF_RB_ROOT, "bpf_rb_root", false }, { BPF_RB_NODE, "bpf_rb_node", false }, - { BPF_REFCOUNT, "bpf_refcount", false }, + { BPF_REFCOUNT, "bpf_refcount", true }, }; int type = 0, i; const char *name = __btf_name_by_offset(btf, var_type->name_off); @@ -3751,7 +3751,7 @@ static int btf_repeat_fields(struct btf_field_info *info, int info_cnt, static int btf_find_struct_field(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, int info_cnt, - u32 level); + u32 level, u32 *seen_mask); /* Find special fields in the struct type of a field. * @@ -3762,7 +3762,7 @@ static int btf_find_struct_field(const struct btf *btf, static int btf_find_nested_struct(const struct btf *btf, const struct btf_type *t, u32 off, u32 nelems, u32 field_mask, struct btf_field_info *info, - int info_cnt, u32 level) + int info_cnt, u32 level, u32 *seen_mask) { int ret, err, i; @@ -3770,7 +3770,7 @@ static int btf_find_nested_struct(const struct btf *btf, const struct btf_type * if (level >= MAX_RESOLVE_DEPTH) return -E2BIG; - ret = btf_find_struct_field(btf, t, field_mask, info, info_cnt, level); + ret = btf_find_struct_field(btf, t, field_mask, info, info_cnt, level, seen_mask); if (ret <= 0) return ret; @@ -3827,7 +3827,7 @@ static int btf_find_field_one(const struct btf *btf, if (expected_size && expected_size != sz * nelems) return 0; ret = btf_find_nested_struct(btf, var_type, off, nelems, field_mask, - &info[0], info_cnt, level); + &info[0], info_cnt, level, seen_mask); return ret; } @@ -3892,11 +3892,11 @@ static int btf_find_field_one(const struct btf *btf, static int btf_find_struct_field(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, int info_cnt, - u32 level) + u32 level, u32 *seen_mask) { int ret, idx = 0; const struct btf_member *member; - u32 i, off, seen_mask = 0; + u32 i, off; for_each_member(i, t, member) { const struct btf_type *member_type = btf_type_by_id(btf, @@ -3910,7 +3910,7 @@ static int btf_find_struct_field(const struct btf *btf, ret = btf_find_field_one(btf, t, member_type, i, off, 0, - field_mask, &seen_mask, + field_mask, seen_mask, &info[idx], info_cnt - idx, level); if (ret < 0) return ret; @@ -3921,11 +3921,11 @@ static int btf_find_struct_field(const struct btf *btf, static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, - int info_cnt, u32 level) + int info_cnt, u32 level, u32 *seen_mask) { int ret, idx = 0; const struct btf_var_secinfo *vsi; - u32 i, off, seen_mask = 0; + u32 i, off; for_each_vsi(i, t, vsi) { const struct btf_type *var = btf_type_by_id(btf, vsi->type); @@ -3933,7 +3933,7 @@ static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t, off = vsi->offset; ret = btf_find_field_one(btf, var, var_type, -1, off, vsi->size, - field_mask, &seen_mask, + field_mask, seen_mask, &info[idx], info_cnt - idx, level); if (ret < 0) @@ -3947,10 +3947,12 @@ static int btf_find_field(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, int info_cnt) { + u32 seen_mask = 0; + if (__btf_type_is_struct(t)) - return btf_find_struct_field(btf, t, field_mask, info, info_cnt, 0); + return btf_find_struct_field(btf, t, field_mask, info, info_cnt, 0, &seen_mask); else if (btf_type_is_datasec(t)) - return btf_find_datasec_var(btf, t, field_mask, info, info_cnt, 0); + return btf_find_datasec_var(btf, t, field_mask, info, info_cnt, 0, &seen_mask); return -EINVAL; } @@ -4168,7 +4170,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type rec->spin_lock_off = rec->fields[i].offset; break; case BPF_RES_SPIN_LOCK: - WARN_ON_ONCE(rec->spin_lock_off >= 0); + WARN_ON_ONCE(rec->res_spin_lock_off >= 0); /* Cache offset for faster lookup at runtime */ rec->res_spin_lock_off = rec->fields[i].offset; break; @@ -7108,7 +7110,7 @@ enum bpf_struct_walk_result { static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, const struct btf_type *t, int off, int size, u32 *next_btf_id, enum bpf_type_flag *flag, - const char **field_name) + const char **field_name, bool walk_flex_arrays) { u32 i, moff, mtrue_end, msize = 0, total_nelems = 0; const struct btf_type *mtype, *elem_type = NULL; @@ -7135,11 +7137,14 @@ again: *flag |= PTR_UNTRUSTED; if (off + size > t->size) { + struct btf_array *array_elem; + + if (!walk_flex_arrays) + goto error; + /* If the last element is a variable size array, we may * need to relax the rule. */ - struct btf_array *array_elem; - if (vlen == 0) goto error; @@ -7404,7 +7409,8 @@ int btf_struct_access(struct bpf_verifier_log *log, t = btf_type_by_id(btf, id); do { - err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag, field_name); + err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag, + field_name, !type_is_alloc(reg->type)); switch (err) { case WALK_PTR: @@ -7463,7 +7469,7 @@ bool btf_types_are_same(const struct btf *btf1, u32 id1, bool btf_struct_ids_match(struct bpf_verifier_log *log, const struct btf *btf, u32 id, int off, const struct btf *need_btf, u32 need_type_id, - bool strict) + bool strict, bool walk_flex_arrays) { const struct btf_type *type; enum bpf_type_flag flag = 0; @@ -7482,7 +7488,8 @@ again: type = btf_type_by_id(btf, id); if (!type) return false; - err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL); + err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL, + walk_flex_arrays); if (err != WALK_STRUCT) return false; diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 4355ccb78a9c67..44068bdd4f759d 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -813,8 +813,10 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *old_prog = NULL; struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; struct bpf_cgroup_storage *new_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; + struct bpf_cgroup_storage *old_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; struct bpf_prog *new_prog = prog ? : link->link.prog; enum cgroup_bpf_attach_type atype; + u32 old_flags, old_pl_flags; struct bpf_prog_list *pl; struct hlist_head *progs; int err; @@ -865,6 +867,8 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, if (pl) { old_prog = pl->prog; + old_pl_flags = pl->flags; + bpf_cgroup_storages_assign(old_storage, pl->storage); } else { pl = kmalloc_obj(*pl); if (!pl) { @@ -884,6 +888,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, pl->link = link; pl->flags = flags; bpf_cgroup_storages_assign(pl->storage, storage); + old_flags = cgrp->bpf.flags[atype]; cgrp->bpf.flags[atype] = saved_flags; if (type == BPF_LSM_CGROUP) { @@ -915,12 +920,15 @@ cleanup: if (old_prog) { pl->prog = old_prog; pl->link = NULL; + pl->flags = old_pl_flags; + bpf_cgroup_storages_assign(pl->storage, old_storage); } bpf_cgroup_storages_free(new_storage); if (!old_prog) { hlist_del(&pl->node); kfree(pl); } + cgrp->bpf.flags[atype] = old_flags; return err; } @@ -1018,6 +1026,20 @@ static void replace_effective_prog(struct cgroup *cgrp, } } +static bool cgroup_bpf_storages_compatible(struct bpf_prog *old_prog, + struct bpf_prog *new_prog) +{ + enum bpf_cgroup_storage_type stype; + + for_each_cgroup_storage_type(stype) { + if (old_prog->aux->cgroup_storage[stype] != + new_prog->aux->cgroup_storage[stype]) + return false; + } + + return true; +} + /** * __cgroup_bpf_replace() - Replace link's program and propagate the change * to descendants @@ -1056,6 +1078,9 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp, if (!found) return -ENOENT; + if (!cgroup_bpf_storages_compatible(link->link.prog, new_prog)) + return -EINVAL; + cgrp->bpf.revisions[atype] += 1; old_prog = xchg(&link->link.prog, new_prog); replace_effective_prog(cgrp, atype, pl); @@ -2235,7 +2260,7 @@ int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level, if (ret < 0) return ret; - if (ctx.optlen > *optlen) + if (ctx.optlen > *optlen || ctx.optlen < 0) return -EFAULT; /* BPF programs can shrink the buffer, export the modifications. diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c index b2a19acadb91ec..4cf120c7b2cb47 100644 --- a/kernel/bpf/const_fold.c +++ b/kernel/bpf/const_fold.c @@ -199,14 +199,9 @@ process_call: ci_out[r] = unknown; break; case BPF_STX: - if (mode != BPF_ATOMIC) - break; - if (insn->imm == BPF_CMPXCHG) - ci_out[BPF_REG_0] = unknown; - else if (insn->imm == BPF_LOAD_ACQ) - *dst = unknown; - else if (insn->imm & BPF_FETCH) - *src = unknown; + r = bpf_atomic_load_reg(insn); + if (r >= 0) + ci_out[r] = unknown; break; } } diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 6e19a030da6f13..883cb7a800b178 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -126,6 +126,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag fp->aux->main_prog_aux = aux; fp->aux->prog = fp; fp->jit_requested = ebpf_jit_enabled(); + fp->jit_required = IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON); fp->blinding_requested = bpf_jit_blinding_enabled(fp); #ifdef CONFIG_CGROUP_BPF aux->cgroup_atype = CGROUP_BPF_ATTACH_TYPE_INVALID; @@ -2670,15 +2671,11 @@ struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct /* In case of BPF to BPF calls, verifier did all the prep * work with regards to JITing, etc. */ - bool jit_needed = false; + bool jit_needed = fp->jit_required; if (fp->bpf_func) goto finalize; - if (IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON) || - bpf_prog_has_kfunc_call(fp)) - jit_needed = true; - if (!bpf_prog_select_interpreter(fp)) jit_needed = true; diff --git a/kernel/bpf/cpumask.c b/kernel/bpf/cpumask.c index b8c805b4b06a0a..1336a4efa75533 100644 --- a/kernel/bpf/cpumask.c +++ b/kernel/bpf/cpumask.c @@ -449,12 +449,12 @@ __bpf_kfunc u32 bpf_cpumask_weight(const struct cpumask *cpumask) * @src__sz: Length of the BPF memory region in bytes. * * Return: - * * 0 if the struct cpumask * instance was populated successfully. + * * 0 if the struct bpf_cpumask * instance was populated successfully. * * -EACCES if the memory region is too small to populate the cpumask. * * -EINVAL if the memory region is not aligned to the size of a long * and the architecture does not support efficient unaligned accesses. */ -__bpf_kfunc int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t src__sz) +__bpf_kfunc int bpf_cpumask_populate(struct bpf_cpumask *cpumask, void *src, size_t src__sz) { unsigned long source = (unsigned long)src; @@ -467,7 +467,7 @@ __bpf_kfunc int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t !IS_ALIGNED(source, sizeof(long))) return -EINVAL; - bitmap_copy(cpumask_bits(cpumask), src, nr_cpu_ids); + bitmap_copy(cpumask_bits(&cpumask->cpumask), src, nr_cpu_ids); return 0; } diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 3cf2cc6e3ab66a..cd42eb838d6c20 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -29,16 +29,7 @@ static int insn_def_regno(const struct bpf_insn *insn) case BPF_ST: return -1; case BPF_STX: - if (BPF_MODE(insn->code) == BPF_ATOMIC || - BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) { - if (insn->imm == BPF_CMPXCHG) - return BPF_REG_0; - else if (insn->imm == BPF_LOAD_ACQ) - return insn->dst_reg; - else if (insn->imm & BPF_FETCH) - return insn->src_reg; - } - return -1; + return bpf_atomic_load_reg(insn); default: return insn->dst_reg; } @@ -1378,7 +1369,6 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) #ifndef CONFIG_BPF_JIT_ALWAYS_ON struct bpf_prog *prog = env->prog; struct bpf_insn *insn = prog->insnsi; - bool has_kfunc_call = bpf_prog_has_kfunc_call(prog); int depth; #endif int i, err = 0; @@ -1404,8 +1394,8 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) return err; } #ifndef CONFIG_BPF_JIT_ALWAYS_ON - if (has_kfunc_call) { - verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); + if (prog->jit_required) { + verbose(env, "program requires BPF JIT compiler but it is not available\n"); return -EINVAL; } for (i = 0; i < env->subprog_cnt; i++) { @@ -1841,8 +1831,10 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) } /* Skip inlining the helper call if the JIT does it. */ - if (bpf_jit_inlines_helper_call(insn->imm)) + if (bpf_jit_inlines_helper_call(insn->imm)) { + prog->jit_required = 1; goto next_insn; + } if (insn->imm == BPF_FUNC_get_route_realm) prog->dst_needed = 1; @@ -2007,6 +1999,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) return -EFAULT; } + if (bpf_map_is_percpu_map(map_ptr->map_type)) + prog->jit_required = true; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -2111,6 +2106,7 @@ patch_map_ops_generic: * way, it's fine to back out this inlining logic */ #ifdef CONFIG_SMP + prog->jit_required = true; insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&cpu_number); insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); insn_buf[2] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, 0); @@ -2132,6 +2128,7 @@ patch_map_ops_generic: /* Implement bpf_get_current_task() and bpf_get_current_task_btf() inline. */ if ((insn->imm == BPF_FUNC_get_current_task || insn->imm == BPF_FUNC_get_current_task_btf) && bpf_verifier_inlines_helper_call(env, insn->imm)) { + prog->jit_required = true; insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)¤t_task); insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0); diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index c18f1e16edee4c..61805d8c95aef3 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -3395,11 +3395,13 @@ __bpf_kfunc void bpf_throw(u64 cookie) WARN_ON_ONCE(!ctx.aux->exception_boundary); WARN_ON_ONCE(!ctx.bp); WARN_ON_ONCE(!ctx.cnt); - /* Prevent KASAN false positives for CONFIG_KASAN_STACK by unpoisoning + /* + * Prevent KASAN false positives for CONFIG_KASAN_STACK by unpoisoning * deeper stack depths than ctx.sp as we do not return from bpf_throw, - * which skips compiler generated instrumentation to do the same. + * which skips compiler generated instrumentation to do the same. Some + * architectures cannot recover sp while unwinding, so fall back to bp. */ - kasan_unpoison_task_stack_below((void *)(long)ctx.sp); + kasan_unpoison_task_stack_below((void *)(long)(ctx.sp ?: ctx.bp)); ctx.aux->bpf_exception_cb(cookie, ctx.sp + ctx.aux->stack_arg_sp_adjust, ctx.bp, 0, 0); WARN(1, "A call to BPF exception callback should never return\n"); } @@ -4388,7 +4390,7 @@ struct bpf_task_work_ctx { struct bpf_map *map; void *map_val; enum task_work_notify_mode mode; - bpf_task_work_callback_t callback_fn; + bpf_callback_t callback_fn; struct rcu_head rcu; } __aligned(8); @@ -4471,7 +4473,8 @@ static void bpf_task_work_callback(struct callback_head *cb) key = (void *)map_key_from_value(ctx->map, ctx->map_val, &idx); migrate_disable(); - ctx->callback_fn(ctx->map, key, ctx->map_val); + ctx->callback_fn((u64)(long)ctx->map, (u64)(long)key, + (u64)(long)ctx->map_val, 0, 0); migrate_enable(); bpf_task_work_ctx_reset(ctx); @@ -4594,7 +4597,7 @@ static struct bpf_task_work_ctx *bpf_task_work_acquire_ctx(struct bpf_task_work } static int bpf_task_work_schedule(struct task_struct *task, struct bpf_task_work *tw, - struct bpf_map *map, bpf_task_work_callback_t callback_fn, + struct bpf_map *map, void *callback_fn, struct bpf_prog_aux *aux, enum task_work_notify_mode mode) { struct bpf_prog *prog; @@ -4619,7 +4622,7 @@ static int bpf_task_work_schedule(struct task_struct *task, struct bpf_task_work } ctx->task = task; - ctx->callback_fn = callback_fn; + ctx->callback_fn = (bpf_callback_t)callback_fn; ctx->prog = prog; ctx->mode = mode; ctx->map = map; diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 0aadfbae0acc5a..da4df129f07d08 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -1213,12 +1213,9 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, clear_stack_for_all_offs(insn, at_out, insn->dst_reg, at_stack_out, sz); - if (insn->imm == BPF_CMPXCHG) - at_out[BPF_REG_0] = none; - else if (insn->imm == BPF_LOAD_ACQ) - *dst = none; - else if (insn->imm & BPF_FETCH) - *src = none; + r = bpf_atomic_load_reg(insn); + if (r >= 0) + at_out[r] = none; } } else if (class == BPF_ST && BPF_MODE(insn->code) == BPF_MEM) { u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); diff --git a/kernel/bpf/mmap_unlock_work.h b/kernel/bpf/mmap_unlock_work.h index 5d18d7d85bef96..1834db20b86124 100644 --- a/kernel/bpf/mmap_unlock_work.h +++ b/kernel/bpf/mmap_unlock_work.h @@ -4,12 +4,15 @@ #ifndef __MMAP_UNLOCK_WORK_H__ #define __MMAP_UNLOCK_WORK_H__ +#include +#include #include /* irq_work to run mmap_read_unlock() in irq_work */ struct mmap_unlock_irq_work { struct irq_work irq_work; struct mm_struct *mm; + atomic_t active; }; DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work); @@ -18,32 +21,36 @@ DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work); * We cannot do mmap_read_unlock() when the irq is disabled, because of * risk to deadlock with rq_lock. To look up vma when the irqs are * disabled, we need to run mmap_read_unlock() in irq_work. We use a - * percpu variable to do the irq_work. If the irq_work is already used - * by another lookup, we fall over. + * percpu variable to do the irq_work. The active flag reserves the slot + * before mmap_read_trylock() and until the irq_work callback consumes mm. */ -static inline bool bpf_mmap_unlock_get_irq_work(struct mmap_unlock_irq_work **work_ptr) +static inline struct mmap_unlock_irq_work *bpf_mmap_unlock_guard_get(void) { - struct mmap_unlock_irq_work *work = NULL; - bool irq_work_busy = false; + struct mmap_unlock_irq_work *work; - if (irqs_disabled()) { - if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { - work = this_cpu_ptr(&mmap_unlock_work); - if (irq_work_is_busy(&work->irq_work)) { - /* cannot queue more up_read, fallback */ - irq_work_busy = true; - } - } else { - /* - * PREEMPT_RT does not allow to trylock mmap sem in - * interrupt disabled context. Force the fallback code. - */ - irq_work_busy = true; - } - } + if (!irqs_disabled()) + return NULL; + + /* + * PREEMPT_RT does not allow to trylock mmap sem in interrupt + * disabled context. Force the fallback code. + */ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + return ERR_PTR(-EBUSY); + + work = this_cpu_ptr(&mmap_unlock_work); + if (irq_work_is_busy(&work->irq_work) || + atomic_cmpxchg_acquire(&work->active, 0, 1)) + return ERR_PTR(-EBUSY); - *work_ptr = work; - return irq_work_busy; + return work; +} + +static inline void +bpf_mmap_unlock_guard_put(struct mmap_unlock_irq_work *work) +{ + if (work) + atomic_set_release(&work->active, 0); } static inline void bpf_mmap_unlock_mm(struct mmap_unlock_irq_work *work, struct mm_struct *mm) diff --git a/kernel/bpf/net_namespace.c b/kernel/bpf/net_namespace.c index 25f30f9edaefd3..81006a242618d8 100644 --- a/kernel/bpf/net_namespace.c +++ b/kernel/bpf/net_namespace.c @@ -171,33 +171,28 @@ static int bpf_netns_link_update_prog(struct bpf_link *link, struct net *net; int idx, ret; + guard(mutex)(&netns_bpf_mutex); + if (old_prog && old_prog != link->prog) return -EPERM; if (new_prog->type != link->prog->type) return -EINVAL; - mutex_lock(&netns_bpf_mutex); - net = net_link->net; - if (!net || !check_net(net)) { + if (!net || !check_net(net)) /* Link auto-detached or netns dying */ - ret = -ENOLINK; - goto out_unlock; - } + return -ENOLINK; run_array = rcu_dereference_protected(net->bpf.run_array[type], lockdep_is_held(&netns_bpf_mutex)); idx = link_index(net, type, net_link); ret = bpf_prog_array_update_at(run_array, idx, new_prog); if (ret) - goto out_unlock; + return ret; old_prog = xchg(&link->prog, new_prog); bpf_prog_put(old_prog); - -out_unlock: - mutex_unlock(&netns_bpf_mutex); - return ret; + return 0; } static int bpf_netns_link_fill_info(const struct bpf_link *link, diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c index 9a5f94371e5065..c1c9dee4dcdd02 100644 --- a/kernel/bpf/queue_stack_maps.c +++ b/kernel/bpf/queue_stack_maps.c @@ -99,8 +99,10 @@ static long __queue_map_get(struct bpf_map *map, void *value, bool delete) int err = 0; void *ptr; - if (raw_res_spin_lock_irqsave(&qs->lock, flags)) + if (raw_res_spin_lock_irqsave(&qs->lock, flags)) { + memset(value, 0, qs->map.value_size); return -EBUSY; + } if (queue_stack_map_is_empty(qs)) { memset(value, 0, qs->map.value_size); @@ -130,8 +132,10 @@ static long __stack_map_get(struct bpf_map *map, void *value, bool delete) void *ptr; u32 index; - if (raw_res_spin_lock_irqsave(&qs->lock, flags)) + if (raw_res_spin_lock_irqsave(&qs->lock, flags)) { + memset(value, 0, qs->map.value_size); return -EBUSY; + } if (queue_stack_map_is_empty(qs)) { memset(value, 0, qs->map.value_size); diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index 35ae64ade36b83..06d3cc192601fc 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -482,7 +482,7 @@ static void *__bpf_ringbuf_reserve(struct bpf_ringbuf *rb, u64 size) prod_pos = rb->producer_pos; new_prod_pos = prod_pos + len; - while (pend_pos < prod_pos) { + while (prod_pos - pend_pos > 0) { hdr = (void *)rb->data + (pend_pos & rb->mask); hdr_len = READ_ONCE(hdr->len); if (hdr_len & BPF_RINGBUF_BUSY_BIT) diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 9b086ecbf15949..142a65672c6f97 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -414,8 +414,7 @@ static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *i static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, u32 trace_nr, bool user, bool may_fault) { - struct mmap_unlock_irq_work *work = NULL; - bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); + struct mmap_unlock_irq_work *work; bool has_user_ctx = user && current && current->mm; struct stack_map_build_id_cache cache = {}; struct vm_area_struct *vma; @@ -426,15 +425,16 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, return; } - /* If the irq_work is in use, fall back to report ips. Same - * fallback is used for kernel stack (!user) on a stackmap with - * build_id. - */ - if (!has_user_ctx || irq_work_busy || !mmap_read_trylock(current->mm)) { - /* cannot access current->mm, fall back to ips */ - for (i = 0; i < trace_nr; i++) - stack_map_build_id_set_ip(&id_offs[i]); - return; + if (!has_user_ctx) + goto fallback; + + work = bpf_mmap_unlock_guard_get(); + if (IS_ERR(work)) + goto fallback; + + if (!mmap_read_trylock(current->mm)) { + bpf_mmap_unlock_guard_put(work); + goto fallback; } for (i = 0; i < trace_nr; i++) { @@ -465,6 +465,12 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, vma->vm_pgoff); } bpf_mmap_unlock_mm(work, current->mm); + return; + +fallback: + /* cannot access current->mm, fall back to ips */ + for (i = 0; i < trace_nr; i++) + stack_map_build_id_set_ip(&id_offs[i]); } static struct perf_callchain_entry * @@ -741,19 +747,55 @@ const struct bpf_func_proto bpf_get_stackid_proto_pe = { .arg3_type = ARG_ANYTHING, }; +static u32 callchain_store(struct perf_callchain_entry *trace, void *buf, + u32 elem_size, u64 flags) +{ + bool user_build_id = flags & BPF_F_USER_BUILD_ID; + u32 skip = flags & BPF_F_SKIP_FIELD_MASK; + u32 trace_nr, copy_len; + u64 *ips; + + trace_nr = trace->nr - skip; + copy_len = trace_nr * elem_size; + + ips = trace->ip + skip; + if (user_build_id) { + struct bpf_stack_build_id *id_offs = buf; + + for (u32 i = 0; i < trace_nr; i++) + id_offs[i].ip = ips[i]; + } else { + memcpy(buf, ips, copy_len); + } + return trace_nr; +} + +static long callchain_finalize(void *buf, u32 size, u32 trace_nr, u32 elem_size, + u64 flags, bool may_fault) +{ + bool user_build_id = flags & BPF_F_USER_BUILD_ID; + bool user = flags & BPF_F_USER_STACK; + u32 copy_len = trace_nr * elem_size; + + if (user_build_id) + stack_map_get_build_id_offset(buf, trace_nr, user, may_fault); + + if (size > copy_len) + memset(buf + copy_len, 0, size - copy_len); + return copy_len; +} + static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, - struct perf_callchain_entry *trace_in, void *buf, u32 size, u64 flags, bool may_fault) { - u32 trace_nr, copy_len, elem_size, max_depth; bool user_build_id = flags & BPF_F_USER_BUILD_ID; bool crosstask = task && task != current; u32 skip = flags & BPF_F_SKIP_FIELD_MASK; bool user = flags & BPF_F_USER_STACK; struct perf_callchain_entry *trace; + u32 trace_nr, elem_size, max_depth; bool kernel = !user; int err = -EINVAL; - u64 *ips; if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | BPF_F_USER_BUILD_ID))) @@ -783,10 +825,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, if (may_fault) rcu_read_lock(); /* need RCU for perf's callchain below */ - if (trace_in) { - trace = trace_in; - trace->nr = min_t(u32, trace->nr, max_depth); - } else if (kernel && task) { + if (kernel && task) { trace = get_callchain_entry_for_task(task, max_depth); } else { trace = get_perf_callchain(regs, kernel, user, max_depth, @@ -800,31 +839,14 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, goto err_fault; } - trace_nr = trace->nr - skip; - copy_len = trace_nr * elem_size; - - ips = trace->ip + skip; - if (user_build_id) { - struct bpf_stack_build_id *id_offs = buf; - u32 i; - - for (i = 0; i < trace_nr; i++) - id_offs[i].ip = ips[i]; - } else { - memcpy(buf, ips, copy_len); - } + trace_nr = callchain_store(trace, buf, elem_size, flags); - /* trace/ips should not be dereferenced after this point */ + /* trace should not be dereferenced after this point */ if (may_fault) rcu_read_unlock(); preempt_enable(); - if (user_build_id) - stack_map_get_build_id_offset(buf, trace_nr, user, may_fault); - - if (size > copy_len) - memset(buf + copy_len, 0, size - copy_len); - return copy_len; + return callchain_finalize(buf, size, trace_nr, elem_size, flags, may_fault); err_fault: err = -EFAULT; @@ -836,7 +858,7 @@ clear: BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size, u64, flags) { - return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */); + return __bpf_get_stack(regs, NULL, buf, size, flags, false /* !may_fault */); } const struct bpf_func_proto bpf_get_stack_proto = { @@ -852,7 +874,7 @@ const struct bpf_func_proto bpf_get_stack_proto = { BPF_CALL_4(bpf_get_stack_sleepable, struct pt_regs *, regs, void *, buf, u32, size, u64, flags) { - return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, true /* may_fault */); + return __bpf_get_stack(regs, NULL, buf, size, flags, true /* may_fault */); } const struct bpf_func_proto bpf_get_stack_sleepable_proto = { @@ -871,14 +893,17 @@ static long __bpf_get_task_stack(struct task_struct *task, void *buf, u32 size, struct pt_regs *regs; long res = -EINVAL; - if (!try_get_task_stack(task)) + if (!try_get_task_stack(task)) { + memset(buf, 0, size); return -EFAULT; + } regs = task_pt_regs(task); if (regs) - res = __bpf_get_stack(regs, task, NULL, buf, size, flags, may_fault); + res = __bpf_get_stack(regs, task, buf, size, flags, may_fault); + else + memset(buf, 0, size); put_task_stack(task); - return res; } @@ -916,6 +941,32 @@ const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = { .arg4_type = ARG_ANYTHING, }; +static int __bpf_get_stack_pe(struct perf_callchain_entry *trace, void *buf, u32 size, + u64 flags) +{ + bool user_build_id = flags & BPF_F_USER_BUILD_ID; + u64 skip = flags & BPF_F_SKIP_FIELD_MASK; + bool user = flags & BPF_F_USER_STACK; + u32 elem_size, max_depth, nr_trace; + bool kernel = !user; + + if (kernel && user_build_id) + return -EINVAL; + + elem_size = user_build_id ? sizeof(struct bpf_stack_build_id) : sizeof(u64); + if (unlikely(size % elem_size)) + return -EINVAL; + + max_depth = stack_map_calculate_max_depth(size, elem_size, flags); + trace->nr = min_t(u32, trace->nr, max_depth); + + if (trace->nr < skip) + return -EFAULT; + + nr_trace = callchain_store(trace, buf, elem_size, flags); + return callchain_finalize(buf, size, nr_trace, elem_size, flags, false /* !may_fault */); +} + BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, void *, buf, u32, size, u64, flags) { @@ -927,7 +978,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, __u64 nr_kernel; if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)) - return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */); + return __bpf_get_stack(regs, NULL, buf, size, flags, false /* !may_fault */); if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | BPF_F_USER_BUILD_ID))) @@ -947,7 +998,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, __u64 nr = trace->nr; trace->nr = nr_kernel; - err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */); + err = __bpf_get_stack_pe(trace, buf, size, flags); /* restore nr */ trace->nr = nr; @@ -957,14 +1008,13 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, skip += nr_kernel; if (skip > BPF_F_SKIP_FIELD_MASK) goto clear; - flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip; - err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */); + err = __bpf_get_stack_pe(trace, buf, size, flags); } - return err; clear: - memset(buf, 0, size); + if (err < 0) + memset(buf, 0, size); return err; } diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index ea2153cf28d0a8..4e6aafad33bd2f 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -812,7 +812,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * infinite loop check triggering, see * iter_active_depths_differ() */ - if (old_reg->iter.btf != cur_reg->iter.btf || + if (old_reg->type != cur_reg->type || + old_reg->iter.btf != cur_reg->iter.btf || old_reg->iter.btf_id != cur_reg->iter.btf_id || old_reg->iter.state != cur_reg->iter.state || /* ignore {old_reg,cur_reg}->iter.depth, see above */ diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 6db306d23b479f..fb678b9dcd3e62 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1657,7 +1657,7 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr, struct bpf_common_at err = security_bpf_map_create(map, attr, token, uattr.is_kernel); if (err) - goto free_map_sec; + goto free_map; err = bpf_map_alloc_id(map); if (err) @@ -3471,9 +3471,10 @@ static const char *bpf_link_type_strs[] = { static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) { const struct bpf_link *link = filp->private_data; - const struct bpf_prog *prog = link->prog; + const struct bpf_prog *prog; enum bpf_link_type type = link->type; char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; + u32 prog_id = 0; if (type < ARRAY_SIZE(bpf_link_type_strs) && bpf_link_type_strs[type]) { if (link->type == BPF_LINK_TYPE_KPROBE_MULTI) @@ -3490,13 +3491,20 @@ static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) } seq_printf(m, "link_id:\t%u\n", link->id); + rcu_read_lock(); + prog = READ_ONCE(link->prog); if (prog) { bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); + prog_id = prog->aux->id; + } + rcu_read_unlock(); + + if (prog) { seq_printf(m, "prog_tag:\t%s\n" "prog_id:\t%u\n", prog_tag, - prog->aux->id); + prog_id); } if (link->ops->show_fdinfo) link->ops->show_fdinfo(link, m); @@ -5535,6 +5543,7 @@ static int bpf_link_get_info_by_fd(struct file *file, { struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info); struct bpf_link_info info; + const struct bpf_prog *prog; u32 info_len = attr->info.info_len; int err; @@ -5549,8 +5558,12 @@ static int bpf_link_get_info_by_fd(struct file *file, info.type = link->type; info.id = link->id; - if (link->prog) - info.prog_id = link->prog->aux->id; + + rcu_read_lock(); + prog = READ_ONCE(link->prog); + if (prog) + info.prog_id = prog->aux->id; + rcu_read_unlock(); if (link->ops->fill_link_info) { err = link->ops->fill_link_info(link, &info); diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index e791ae065c39bf..13e1aabe6f8868 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -753,9 +753,9 @@ static struct bpf_iter_reg task_vma_reg_info = { BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, bpf_callback_t, callback_fn, void *, callback_ctx, u64, flags) { - struct mmap_unlock_irq_work *work = NULL; + struct mmap_unlock_irq_work *work; struct vm_area_struct *vma; - bool irq_work_busy = false; + bool __maybe_unused mmput_needed = false; struct mm_struct *mm; int ret = -ENOENT; @@ -765,14 +765,43 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, if (!task) return -ENOENT; - mm = task->mm; + if (task == current) { + mm = task->mm; + } else { + /* + * Foreign task: pin task->mm against a concurrent exit_mm(). + * Use trylock on alloc_lock instead of get_task_mm()'s + * blocking task_lock() to avoid deadlocking the target task. + */ + if (!IS_ENABLED(CONFIG_MMU)) + return -EOPNOTSUPP; + if (irqs_disabled()) + return -EBUSY; + if (!spin_trylock(&task->alloc_lock)) + return -EBUSY; + mm = task->mm; + if (mm && !(task->flags & PF_KTHREAD)) { + mmget(mm); + mmput_needed = true; + } else { + mm = NULL; + } + spin_unlock(&task->alloc_lock); + } if (!mm) return -ENOENT; - irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); + work = bpf_mmap_unlock_guard_get(); + if (IS_ERR(work)) { + ret = PTR_ERR(work); + goto out; + } - if (irq_work_busy || !mmap_read_trylock(mm)) - return -EBUSY; + if (!mmap_read_trylock(mm)) { + bpf_mmap_unlock_guard_put(work); + ret = -EBUSY; + goto out; + } vma = find_vma(mm, start); @@ -782,6 +811,11 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, ret = 0; } bpf_mmap_unlock_mm(work, mm); +out: +#ifdef CONFIG_MMU + if (mmput_needed) + mmput_async(mm); +#endif return ret; } @@ -1161,6 +1195,8 @@ static void do_mmap_read_unlock(struct irq_work *entry) work = container_of(entry, struct mmap_unlock_irq_work, irq_work); mmap_read_unlock_non_owner(work->mm); + work->mm = NULL; + bpf_mmap_unlock_guard_put(work); } static int __init task_iter_init(void) diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 1a721fc4bef562..02c208bd2b6377 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -670,6 +670,13 @@ out: return ERR_PTR(err); } +void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) +{ + trampoline_lock(tr); + tr->flags |= flags; + trampoline_unlock(tr); +} + static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex, const struct bpf_trampoline_ops *ops, void *data) { @@ -1536,6 +1543,7 @@ static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_ima if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); + tr->func.ftrace_managed = true; ftrace_hash_add(data->reg, data->entry, ip, addr); tr->cur_image = im; return 0; @@ -1584,7 +1592,17 @@ static void bpf_trampoline_multi_attach_init(struct bpf_trampoline *tr) static void bpf_trampoline_multi_attach_free(struct bpf_trampoline *tr) { - if (tr->multi_attach.old_image) + /* + * Only free old_image if it is no longer the active image. + * When bpf_trampoline_update() fails before modify_fentry_multi()/ + * unregister_fentry_multi() is called, cur_image is unchanged + * (cur_image == old_image) and ftrace still points to it. Freeing + * it would cause a UAF when ftrace calls into the freed memory. + * On success, cur_image is either a new image or NULL, so + * old_image != cur_image means the image is stale. + */ + if (tr->multi_attach.old_image && + tr->multi_attach.old_image != tr->cur_image) bpf_tramp_image_put(tr->multi_attach.old_image); tr->multi_attach.old_image = NULL; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index fdc5fbb1f78ca7..14a9fd2d547200 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -204,6 +204,7 @@ static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int par static int release_reference_nomark(struct bpf_verifier_state *state, int id); static int release_reference(struct bpf_verifier_env *env, int id); static void invalidate_non_owning_refs(struct bpf_verifier_env *env); +static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env); static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env); static bool is_tracing_prog_type(enum bpf_prog_type type); static int ref_set_non_owning(struct bpf_verifier_env *env, @@ -2715,6 +2716,8 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) prog_aux->kfunc_tab = tab; } + env->prog->jit_required = 1; + /* func_id == 0 is always invalid, but instead of returning an error, be * conservative and wait until the code elimination pass before returning * error, so that invalid calls that get pruned out can be in BPF programs @@ -2769,11 +2772,6 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) return 0; } -bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) -{ - return !!prog->aux->kfunc_tab; -} - static int add_subprog_and_kfunc(struct bpf_verifier_env *env) { struct bpf_subprog_info *subprog = env->subprog_info; @@ -4349,7 +4347,8 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, */ if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->var_off.value, kptr_field->kptr.btf, kptr_field->kptr.btf_id, - kptr_field->type != BPF_KPTR_UNREF)) + kptr_field->type != BPF_KPTR_UNREF, + !type_is_alloc(reg->type))) goto bad_type; return 0; bad_type: @@ -4844,6 +4843,30 @@ static bool is_arena_reg(struct bpf_verifier_env *env, int regno) return reg->type == PTR_TO_ARENA; } +static bool is_load_acq_unsafe(struct bpf_verifier_env *env, int regno, + struct bpf_insn *insn) +{ + const struct bpf_reg_state *reg = reg_state(env, regno); + + /* + * A BPF_LOAD_ACQ is not rewritten to a BPF_PROBE_MEM load by the + * verifier, unlike a regular BPF_LDX. The JIT would emit a plain load + * with no exception table entry, so a fault (e.g. NULL deref) crashes + * the kernel instead of being handled. + * + * Reject the source pointer types that a BPF_LDX would have had that + * fault protection applied to, i.e. the ones bpf_convert_ctx_accesses() + * turns into BPF_PROBE_MEM: a bare PTR_TO_BTF_ID and any PTR_UNTRUSTED + * pointer (untrusted btf ids, untrusted MEM_ALLOC, rdonly untrusted + * memory). A PTR_TRUSTED pointer is not among them, is not converted, + * and stays allowed. Same for the other flagged PTR_TO_BTF_ID variants + * (MEM_ALLOC, MEM_RCU, ...), hence the exact match on the base type. + */ + return insn->imm == BPF_LOAD_ACQ && + (reg->type == PTR_TO_BTF_ID || + (type_flag(reg->type) & PTR_UNTRUSTED)); +} + /* Return false if @regno contains a pointer whose type isn't supported for * atomic instruction @insn. */ @@ -4860,7 +4883,8 @@ static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno, return false; if (is_arena_reg(env, regno)) return bpf_jit_supports_insn(insn, true); - + if (is_load_acq_unsafe(env, regno, insn)) + return false; return true; } @@ -5213,8 +5237,8 @@ continue_func: if (!priv_stack_supported) subprog[idx].priv_stack_mode = NO_PRIV_STACK; - if (subprog[idx].has_tail_call) - tail_call_reachable = true; + /* sync tail_call_reachable with callee state on entry */ + tail_call_reachable = subprog[idx].has_tail_call; frame = bpf_subprog_is_global(env, idx) ? 0 : frame + 1; if (frame >= MAX_CALL_FRAMES) { @@ -5789,6 +5813,11 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, return -EACCES; } + if (atype != BPF_READ && (type_flag(reg->type) & PTR_UNTRUSTED)) { + verbose(env, "only read is supported\n"); + return -EACCES; + } + if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) { if (!btf_is_kernel(reg->btf)) { verifier_bug(env, "reg->btf must be kernel btf"); @@ -5801,8 +5830,7 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, reg_arg_name(env, argno), tname, off, size); } else { /* Writes are permitted with default btf_struct_access for - * program allocated objects (which always have id > 0), - * but not for untrusted PTR_TO_BTF_ID | MEM_ALLOC. + * program allocated objects (which always have id > 0). */ if (atype != BPF_READ && !type_is_ptr_alloc_obj(reg->type)) { verbose(env, "only read is supported\n"); @@ -6326,11 +6354,23 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && regs[value_regno].type == SCALAR_VALUE) { - if (!is_ldsx) + if (!is_ldsx) { /* b/h/w load zero-extends, mark upper bits as known 0 */ coerce_reg_to_size(®s[value_regno], size); - else + } else { + /* + * Sign-extension can change the register value relative + * to a scalar it is linked with by id (e.g. a zero- + * extending fill of the same spilled stack slot), thus + * drop the shared id in that case. + */ + bool no_sext = reg_umax(®s[value_regno]) < + (1ULL << (size * BITS_PER_BYTE - 1)); + coerce_reg_to_size_sx(®s[value_regno], size); + if (!no_sext) + clear_scalar_id(®s[value_regno]); + } } return err; } @@ -6467,21 +6507,12 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return -EACCES; } - if (insn->imm & BPF_FETCH) { - if (insn->imm == BPF_CMPXCHG) - load_reg = BPF_REG_0; - else - load_reg = insn->src_reg; - + load_reg = bpf_atomic_load_reg(insn); + if (load_reg >= 0) { /* check and record load of old value */ err = check_reg_arg(env, load_reg, DST_OP); if (err) return err; - } else { - /* This instruction accesses a memory location but doesn't - * actually load it into a register. - */ - load_reg = -1; } dst_reg = cur_regs(env) + insn->dst_reg; @@ -6498,11 +6529,9 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, if (err) return err; - if (is_arena_reg(env, insn->dst_reg)) { - err = save_aux_ptr_type(env, PTR_TO_ARENA, false); - if (err) - return err; - } + err = save_aux_ptr_type(env, dst_reg->type, false); + if (err) + return err; /* Check whether we can write into the same memory. */ err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, true, false); @@ -6516,7 +6545,7 @@ static int check_atomic_load(struct bpf_verifier_env *env, { int err; - err = check_load_mem(env, insn, true, false, false, "atomic_load"); + err = check_reg_arg(env, insn->src_reg, SRC_OP); if (err) return err; @@ -6527,7 +6556,7 @@ static int check_atomic_load(struct bpf_verifier_env *env, return -EACCES; } - return 0; + return check_load_mem(env, insn, true, false, false, "atomic_load"); } static int check_atomic_store(struct bpf_verifier_env *env, @@ -7051,6 +7080,7 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state return err; } } else { + bool was_in_rcu_cs; void *ptr; int type; @@ -7078,10 +7108,13 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state verbose(env, "%s_unlock cannot be out of order\n", lock_str); return -EINVAL; } + was_in_rcu_cs = in_rcu_cs(env); if (release_lock_state(cur, type, reg->id, ptr)) { verbose(env, "%s_unlock of different lock\n", lock_str); return -EINVAL; } + if (was_in_rcu_cs && !in_rcu_cs(env)) + invalidate_rcu_protected_refs(env); invalidate_non_owning_refs(env); } @@ -7947,7 +7980,7 @@ found: if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->var_off.value, btf_vmlinux, *arg_btf_id, - strict_type_match)) { + strict_type_match, !type_is_alloc(reg->type))) { verbose(env, "%s is of type %s but %s is expected\n", reg_arg_name(env, argno), btf_type_name(reg->btf, reg->btf_id), @@ -11414,7 +11447,8 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, ®_ref_id); reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off); struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->var_off.value, - meta->btf, ref_id, strict_type_match); + meta->btf, ref_id, strict_type_match, + !type_is_alloc(reg->type)); /* If kfunc is accepting a projection type (ie. __sk_buff), it cannot * actually use it -- it must cast to the underlying type. So we allow * caller to pass in the underlying type. @@ -11861,7 +11895,8 @@ __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, et = btf_type_by_id(field->graph_root.btf, field->graph_root.value_btf_id); t = btf_type_by_id(reg->btf, reg->btf_id); if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf, - field->graph_root.value_btf_id, true)) { + field->graph_root.value_btf_id, true, + !type_is_alloc(reg->type))) { verbose(env, "operation on %s expects arg#1 %s at offset=%d " "in struct %s, but arg is at offset=%d in struct %s\n", btf_field_type_name(head_field_type), @@ -12033,6 +12068,11 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } meta->r0_size = reg->var_off.value; + if (meta->r0_size > U32_MAX) { + verbose(env, "%s rdonly/rdwr_buf_size exceeds u32 max\n", + reg_arg_name(env, argno)); + return -EINVAL; + } if (regno >= 0) ret = mark_chain_precision(env, regno); else @@ -17830,6 +17870,7 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, return -EOPNOTSUPP; } env->prog->aux->arena = (void *)map; + env->prog->jit_required = true; if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) { verbose(env, "arena's user address must be set via map_extra or mmap()\n"); return -EINVAL; @@ -17877,6 +17918,7 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) return err; } env->insn_array_maps[env->insn_array_map_cnt++] = map; + env->prog->jit_required = true; } return env->used_map_cnt - 1; @@ -18838,6 +18880,9 @@ static int btf_id_allow_sleepable(u32 btf_id, unsigned long addr, const struct b const struct btf_type *t; const char *tname; + if (!btf_is_kernel(btf)) + return -EINVAL; + switch (prog->type) { case BPF_PROG_TYPE_TRACING: t = btf_type_by_id(btf, btf_id); @@ -18922,6 +18967,20 @@ btf_attach_func_proto(struct bpf_verifier_log *log, struct btf *btf, u32 func_id return btf_type_by_id(btf, func->type); } +static bool attach_uses_trampoline_retval(enum bpf_attach_type type) +{ + switch (type) { + case BPF_MODIFY_RETURN: + case BPF_TRACE_FEXIT: + case BPF_TRACE_FEXIT_MULTI: + case BPF_TRACE_FSESSION: + case BPF_TRACE_FSESSION_MULTI: + return true; + default: + return false; + } +} + int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *prog, const struct bpf_prog *tgt_prog, @@ -19186,6 +19245,14 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, if (ret < 0) return ret; + if (tgt_info->fmodel.ret_size > 8 && + attach_uses_trampoline_retval(prog->expected_attach_type)) { + bpf_log(log, + "Attach to function %s with a >8 byte return value is not supported for this attach type\n", + tname); + return -EOPNOTSUPP; + } + /* * *.multi programs don't need an address during program * verification, we just take the module ref if needed. @@ -19421,7 +19488,7 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) return -ENOMEM; if (tgt_prog && tgt_prog->aux->tail_call_reachable) - tr->flags = BPF_TRAMP_F_TAIL_CALL_CTX; + bpf_trampoline_set_flags(tr, BPF_TRAMP_F_TAIL_CALL_CTX); prog->aux->dst_trampoline = tr; return 0; @@ -19460,6 +19527,9 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt err = btf_distill_func_proto(NULL, btf, t, tname, &tgt_info->fmodel); if (err < 0) return err; + if (tgt_info->fmodel.ret_size > 8 && + attach_uses_trampoline_retval(prog->expected_attach_type)) + return -EOPNOTSUPP; if (btf_is_module(btf)) { /* The bpf program already holds reference to module. */ if (WARN_ON_ONCE(!prog->aux->mod)) @@ -19483,13 +19553,25 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt struct btf *bpf_get_btf_vmlinux(void) { - if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { + /* Pairs with the smp_store_release() on the parse path below. */ + struct btf *btf = smp_load_acquire(&btf_vmlinux); + + if (!btf && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { mutex_lock(&bpf_verifier_lock); - if (!btf_vmlinux) - btf_vmlinux = btf_parse_vmlinux(); + btf = btf_vmlinux; + if (!btf) { + btf = btf_parse_vmlinux(); + /* + * Order the parsed BTF contents and the globals the + * parse populated (e.g. bpf_ctx_convert.t) before + * the pointer publication. Pairs with the acquire + * on the lockless fast path above. + */ + smp_store_release(&btf_vmlinux, btf); + } mutex_unlock(&bpf_verifier_lock); } - return btf_vmlinux; + return btf; } /* diff --git a/kernel/cgroup/cpuset-internal.h b/kernel/cgroup/cpuset-internal.h index f7aaf01f7cd5e3..140700e5e236d7 100644 --- a/kernel/cgroup/cpuset-internal.h +++ b/kernel/cgroup/cpuset-internal.h @@ -165,7 +165,7 @@ struct cpuset { * number of SCHED_DEADLINE tasks attached to this cpuset, so that we * know when to rebuild associated root domain bandwidth information. */ - int nr_deadline_tasks; + atomic_t nr_deadline_tasks; int nr_migrate_dl_tasks; /* DL bandwidth that needs destination reservation for this attach. */ u64 sum_migrate_dl_bw; diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 45944b3e31ca49..1c6d568006688b 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -222,14 +222,14 @@ void inc_dl_tasks_cs(struct task_struct *p) { struct cpuset *cs = task_cs(p); - cs->nr_deadline_tasks++; + atomic_inc(&cs->nr_deadline_tasks); } void dec_dl_tasks_cs(struct task_struct *p) { struct cpuset *cs = task_cs(p); - cs->nr_deadline_tasks--; + atomic_dec(&cs->nr_deadline_tasks); } static inline bool is_partition_valid(const struct cpuset *cs) @@ -918,7 +918,7 @@ static void dl_update_tasks_root_domain(struct cpuset *cs) struct css_task_iter it; struct task_struct *task; - if (cs->nr_deadline_tasks == 0) + if (atomic_read(&cs->nr_deadline_tasks) == 0) return; css_task_iter_start(&cs->css, 0, &it); @@ -1089,12 +1089,35 @@ void cpuset_update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus) * @cs: the cpuset the need to recompute the new effective_cpus mask * @parent: the parent cpuset * + * For v2, the parent's effective_cpus is inherited if cpumask is empty. * The result is valid only if the given cpuset isn't a partition root. */ static void compute_effective_cpumask(struct cpumask *new_cpus, struct cpuset *cs, struct cpuset *parent) { - cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus); + bool has_cpus; + + has_cpus = cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus); + if (!has_cpus && is_in_v2_mode()) + cpumask_copy(new_cpus, parent->effective_cpus); +} + +/** + * compute_effective_nodemask - Compute the effective nodemask of the cpuset + * @new_mems: the temp variable for the new effective_mems mask + * @cs: the cpuset the need to recompute the new effective_mems mask + * @parent: the parent cpuset + * + * For v2, the parent's effective_mems is inherited if nodemask is empty. + */ +static void compute_effective_nodemask(nodemask_t *new_mems, + struct cpuset *cs, struct cpuset *parent) +{ + bool has_mems; + + has_mems = nodes_and(*new_mems, cs->mems_allowed, parent->effective_mems); + if (!has_mems && is_in_v2_mode()) + nodes_copy(*new_mems, parent->effective_mems); } /* @@ -1525,7 +1548,7 @@ static int remote_partition_enable(struct cpuset *cs, int new_prs, cpumask_copy(cs->effective_xcpus, tmp->new_cpus); spin_unlock_irq(&callback_lock); cpuset_force_rebuild(); - cs->prs_err = 0; + WRITE_ONCE(cs->prs_err, 0); /* * Propagate changes in top_cpuset's effective_cpus down the hierarchy. @@ -1599,7 +1622,7 @@ static void remote_cpus_update(struct cpuset *cs, struct cpumask *xcpus, WARN_ON_ONCE(!cpumask_subset(cs->effective_xcpus, subpartitions_cpus)); if (cpumask_empty(excpus)) { - cs->prs_err = PERR_CPUSEMPTY; + WRITE_ONCE(cs->prs_err, PERR_CPUSEMPTY); goto invalidate; } @@ -1614,13 +1637,13 @@ static void remote_cpus_update(struct cpuset *cs, struct cpumask *xcpus, if (adding) { WARN_ON_ONCE(cpumask_intersects(tmp->addmask, subpartitions_cpus)); if (!capable(CAP_SYS_ADMIN)) - cs->prs_err = PERR_ACCESS; + WRITE_ONCE(cs->prs_err, PERR_ACCESS); else if (cpumask_intersects(tmp->addmask, subpartitions_cpus) || cpumask_subset(top_cpuset.effective_cpus, tmp->addmask)) - cs->prs_err = PERR_NOCPUS; + WRITE_ONCE(cs->prs_err, PERR_NOCPUS); else if ((prs == PRS_ISOLATED) && !isolated_cpus_can_update(tmp->addmask, tmp->delmask)) - cs->prs_err = PERR_HKEEPING; + WRITE_ONCE(cs->prs_err, PERR_HKEEPING); if (cs->prs_err) goto invalidate; } @@ -2048,13 +2071,13 @@ static void compute_partition_effective_cpumask(struct cpuset *cs, * partition root. */ WARN_ON_ONCE(is_remote_partition(child)); - child->prs_err = 0; + WRITE_ONCE(child->prs_err, 0); if (!cpumask_subset(child->effective_xcpus, cs->effective_xcpus)) - child->prs_err = PERR_INVCPUS; + WRITE_ONCE(child->prs_err, PERR_INVCPUS); else if (populated && cpumask_subset(new_ecpus, child->effective_xcpus)) - child->prs_err = PERR_NOCPUS; + WRITE_ONCE(child->prs_err, PERR_NOCPUS); if (child->prs_err) { int old_prs = child->partition_root_state; @@ -2143,15 +2166,6 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp, goto update_parent_effective; } - /* - * If it becomes empty, inherit the effective mask of the - * parent, which is guaranteed to have some CPUs unless - * it is a partition root that has explicitly distributed - * out all its CPUs. - */ - if (is_in_v2_mode() && !remote && cpumask_empty(tmp->new_cpus)) - cpumask_copy(tmp->new_cpus, parent->effective_cpus); - /* * Skip the whole subtree if * 1) the cpumask remains the same, @@ -2367,8 +2381,10 @@ static void partition_cpus_change(struct cpuset *cs, struct cpuset *trialcs, return; prs_err = validate_partition(cs, trialcs); - if (prs_err) - trialcs->prs_err = cs->prs_err = prs_err; + if (prs_err) { + WRITE_ONCE(cs->prs_err, prs_err); + trialcs->prs_err = prs_err; + } if (is_remote_partition(cs)) { if (trialcs->prs_err) @@ -2697,14 +2713,7 @@ static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems) cpuset_for_each_descendant_pre(cp, pos_css, cs) { struct cpuset *parent = parent_cs(cp); - bool has_mems = nodes_and(*new_mems, cp->mems_allowed, parent->effective_mems); - - /* - * If it becomes empty, inherit the effective mask of the - * parent, which is guaranteed to have some MEMs. - */ - if (is_in_v2_mode() && !has_mems) - *new_mems = parent->effective_mems; + compute_effective_nodemask(new_mems, cp, parent); /* Skip the whole subtree if the nodemask remains the same. */ if (nodes_equal(*new_mems, cp->effective_mems)) { @@ -3209,8 +3218,8 @@ out: cs->old_mems_allowed = cpuset_attach_nodemask_to; if (cs->nr_migrate_dl_tasks) { - cs->nr_deadline_tasks += cs->nr_migrate_dl_tasks; - oldcs->nr_deadline_tasks -= cs->nr_migrate_dl_tasks; + atomic_add(cs->nr_migrate_dl_tasks, &cs->nr_deadline_tasks); + atomic_sub(cs->nr_migrate_dl_tasks, &oldcs->nr_deadline_tasks); reset_migrate_dl_data(cs); } @@ -3778,7 +3787,7 @@ retry: parent = parent_cs(cs); compute_effective_cpumask(&new_cpus, cs, parent); - nodes_and(new_mems, cs->mems_allowed, parent->effective_mems); + compute_effective_nodemask(&new_mems, cs, parent); if (!tmp || !cs->partition_root_state) goto update_tasks; @@ -3794,7 +3803,7 @@ retry: if (remote && (cpumask_empty(subpartitions_cpus) || (cpumask_empty(&new_cpus) && partition_is_populated(cs, NULL)))) { - cs->prs_err = PERR_HOTPLUG; + WRITE_ONCE(cs->prs_err, PERR_HOTPLUG); remote_partition_disable(cs, tmp); compute_effective_cpumask(&new_cpus, cs, parent); remote = false; diff --git a/kernel/crash_dump_dm_crypt.c b/kernel/crash_dump_dm_crypt.c index cb875ddb6ba68b..c685497cd470e5 100644 --- a/kernel/crash_dump_dm_crypt.c +++ b/kernel/crash_dump_dm_crypt.c @@ -81,7 +81,6 @@ static int add_key_to_keyring(struct dm_crypt_key *dm_key, kexec_dprintk("Error when adding key"); } - key_ref_put(keyring_ref); return r; } @@ -104,6 +103,7 @@ static int restore_dm_crypt_keys_to_thread_keyring(void) struct dm_crypt_key *key; size_t keys_header_size; key_ref_t keyring_ref; + int ret = 0; u64 addr; /* find the target keyring (which must be writable) */ @@ -118,7 +118,8 @@ static int restore_dm_crypt_keys_to_thread_keyring(void) dm_crypt_keys_read((char *)&key_count, sizeof(key_count), &addr); if (key_count > KEY_NUM_MAX) { kexec_dprintk("Failed to read the number of dm-crypt keys\n"); - return -1; + ret = -1; + goto out; } kexec_dprintk("There are %u keys\n", key_count); @@ -126,8 +127,10 @@ static int restore_dm_crypt_keys_to_thread_keyring(void) keys_header_size = get_keys_header_size(key_count); keys_header = kzalloc(keys_header_size, GFP_KERNEL); - if (!keys_header) - return -ENOMEM; + if (!keys_header) { + ret = -ENOMEM; + goto out; + } dm_crypt_keys_read((char *)keys_header, keys_header_size, &addr); @@ -137,7 +140,9 @@ static int restore_dm_crypt_keys_to_thread_keyring(void) add_key_to_keyring(key, keyring_ref); } - return 0; +out: + key_ref_put(keyring_ref); + return ret; } static int read_key_from_user_keyring(struct dm_crypt_key *dm_key) diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 1abd3e6146f450..6e8db52866bffe 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -266,9 +266,9 @@ void __init swiotlb_update_mem_attributes(void) } static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start, - unsigned long nslabs, bool late_alloc, unsigned int nareas) + void *vaddr, unsigned long nslabs, bool late_alloc, + unsigned int nareas) { - void *vaddr = phys_to_virt(start); unsigned long bytes = nslabs << IO_TLB_SHIFT, i; mem->nslabs = nslabs; @@ -409,7 +409,7 @@ void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags, return; } - swiotlb_init_io_tlb_pool(mem, __pa(tlb), nslabs, false, nareas); + swiotlb_init_io_tlb_pool(mem, __pa(tlb), tlb, nslabs, false, nareas); add_mem_pool(&io_tlb_default_mem, mem); if (flags & SWIOTLB_VERBOSE) @@ -507,7 +507,7 @@ retry: set_memory_decrypted((unsigned long)vstart, (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT); - swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), nslabs, true, + swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), vstart, nslabs, true, nareas); add_mem_pool(&io_tlb_default_mem, mem); @@ -605,25 +605,26 @@ error: * @bytes: Size of the buffer. * @phys_limit: Maximum allowed physical address of the buffer. * @gfp: GFP flags for the allocation. + * @vaddr: Receives the virtual address for the allocated buffer. * * Return: Allocated pages, or %NULL on allocation failure. */ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes, - u64 phys_limit, gfp_t gfp) + u64 phys_limit, gfp_t gfp, void **vaddr) { struct page *page; + *vaddr = NULL; + /* * Allocate from the atomic pools if memory is encrypted and * the allocation is atomic, because decrypting may block. */ if (!gfpflags_allow_blocking(gfp) && dev && force_dma_unencrypted(dev)) { - void *vaddr; - if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL)) return NULL; - return dma_alloc_from_pool(dev, bytes, &vaddr, gfp, + return dma_alloc_from_pool(dev, bytes, vaddr, gfp, dma_coherent_ok); } @@ -645,6 +646,8 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes, return NULL; } + if (page) + *vaddr = phys_to_virt(page_to_phys(page)); return page; } @@ -685,6 +688,7 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev, { struct io_tlb_pool *pool; unsigned int slot_order; + void *tlb_vaddr; struct page *tlb; size_t pool_size; size_t tlb_size; @@ -701,7 +705,8 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev, pool->areas = (void *)pool + sizeof(*pool); tlb_size = nslabs << IO_TLB_SHIFT; - while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp))) { + while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp, + &tlb_vaddr))) { if (nslabs <= minslabs) goto error_tlb; nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE); @@ -715,11 +720,12 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev, if (!pool->slots) goto error_slots; - swiotlb_init_io_tlb_pool(pool, page_to_phys(tlb), nslabs, true, nareas); + swiotlb_init_io_tlb_pool(pool, page_to_phys(tlb), tlb_vaddr, nslabs, + true, nareas); return pool; error_slots: - swiotlb_free_tlb(page_address(tlb), tlb_size); + swiotlb_free_tlb(tlb_vaddr, tlb_size); error_tlb: kfree(pool); error: @@ -1851,7 +1857,8 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem, set_memory_decrypted((unsigned long)phys_to_virt(rmem->base), rmem->size >> PAGE_SHIFT); - swiotlb_init_io_tlb_pool(pool, rmem->base, nslabs, + swiotlb_init_io_tlb_pool(pool, rmem->base, phys_to_virt(rmem->base), + nslabs, false, nareas); mem->force_bounce = true; mem->for_alloc = true; diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 6300b216012cbe..e4f526c9bfbf04 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -1806,14 +1806,6 @@ static struct xol_area *get_xol_area(void) return area; } -void __weak arch_uprobe_clear_state(struct mm_struct *mm) -{ -} - -void __weak arch_uprobe_init_state(struct mm_struct *mm) -{ -} - /* * uprobe_clear_state - Free the area allocated for slots. */ @@ -1825,8 +1817,6 @@ void uprobe_clear_state(struct mm_struct *mm) delayed_uprobe_remove(NULL, mm); mutex_unlock(&delayed_uprobe_lock); - arch_uprobe_clear_state(mm); - if (!area) return; diff --git a/kernel/fork.c b/kernel/fork.c index f0e2e131a9a5af..abc2f01ac35731 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1069,7 +1069,6 @@ static void mm_init_uprobes_state(struct mm_struct *mm) { #ifdef CONFIG_UPROBES mm->uprobes_state.xol_area = NULL; - arch_uprobe_init_state(mm); #endif } diff --git a/kernel/kcsan/core.c b/kernel/kcsan/core.c index 8a7baf4e332e36..2db82661cd60a1 100644 --- a/kernel/kcsan/core.c +++ b/kernel/kcsan/core.c @@ -585,8 +585,14 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned * information is lost if dirtied by KCSAN. */ kcsan_save_irqtrace(current); - if (!interrupt_watcher) + if (!interrupt_watcher) { local_irq_save(irq_flags); + /* + * NMIs can still fire, disable checking for all interrupt + * contexts. + */ + raw_cpu_ptr(&kcsan_cpu_ctx)->disable_count++; + } watchpoint = insert_watchpoint((unsigned long)ptr, size, is_write); if (watchpoint == NULL) { @@ -699,8 +705,10 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned atomic_long_dec(&kcsan_counters[KCSAN_COUNTER_USED_WATCHPOINTS]); out_unlock: - if (!interrupt_watcher) + if (!interrupt_watcher) { + raw_cpu_ptr(&kcsan_cpu_ctx)->disable_count--; local_irq_restore(irq_flags); + } kcsan_restore_irqtrace(current); ctx->disable_scoped--; diff --git a/kernel/liveupdate/luo_flb.c b/kernel/liveupdate/luo_flb.c index 5c27134ce7ba57..cd715a7c1d9929 100644 --- a/kernel/liveupdate/luo_flb.c +++ b/kernel/liveupdate/luo_flb.c @@ -133,7 +133,7 @@ static int luo_flb_file_preserve_one(struct liveupdate_flb *flb) return 0; } -static void luo_flb_file_unpreserve_one(struct liveupdate_flb *flb) +void liveupdate_flb_put_outgoing(struct liveupdate_flb *flb) { struct luo_flb_private *private = luo_flb_get_private(flb); @@ -168,7 +168,10 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb) if (private->incoming.finished) return -ENODATA; - if (private->incoming.retrieved) + if (private->incoming.retrieve_status < 0) + return private->incoming.retrieve_status; + + if (private->incoming.retrieve_status > 0) return 0; if (!fh->active) @@ -194,12 +197,13 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb) err = flb->ops->retrieve(&args); if (err) { + private->incoming.retrieve_status = err; module_put(flb->ops->owner); return err; } private->incoming.obj = args.obj; - private->incoming.retrieved = true; + private->incoming.retrieve_status = 1; return 0; } @@ -213,7 +217,7 @@ void liveupdate_flb_put_incoming(struct liveupdate_flb *flb) if (!refcount_dec_and_test(&private->incoming.count)) return; - if (!private->incoming.retrieved) { + if (private->incoming.retrieve_status <= 0) { int err = luo_flb_retrieve_one(flb); if (WARN_ON(err)) @@ -264,7 +268,7 @@ int luo_flb_file_preserve(struct liveupdate_file_handler *fh) exit_err: list_for_each_entry_continue_reverse(iter, flb_list, list) - luo_flb_file_unpreserve_one(iter->flb); + liveupdate_flb_put_outgoing(iter->flb); up_read(&luo_register_rwlock); return err; @@ -289,7 +293,7 @@ void luo_flb_file_unpreserve(struct liveupdate_file_handler *fh) guard(rwsem_read)(&luo_register_rwlock); list_for_each_entry_reverse(iter, flb_list, list) - luo_flb_file_unpreserve_one(iter->flb); + liveupdate_flb_put_outgoing(iter->flb); } /** @@ -544,6 +548,10 @@ int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp) return -EOPNOTSUPP; guard(mutex)(&private->outgoing.lock); + if (!private->outgoing.obj) + return -ENOENT; + + refcount_inc(&private->outgoing.count); *objp = private->outgoing.obj; return 0; diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c index f38b5b18f3f818..b4a9f55c44987a 100644 --- a/kernel/liveupdate/luo_session.c +++ b/kernel/liveupdate/luo_session.c @@ -316,8 +316,12 @@ static int luo_session_finish(struct luo_session *session, struct luo_ucmd *ucmd) { struct liveupdate_session_finish *argp = ucmd->cmd; - int err = luo_session_finish_one(session); + int err; + + if (argp->reserved) + return -EINVAL; + err = luo_session_finish_one(session); if (err) return err; diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 2d4c5bab5af887..e0de811148242a 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -5437,6 +5437,8 @@ __lock_set_class(struct lockdep_map *lock, const char *name, lock->wait_type_outer, lock->lock_type); class = register_lock_class(lock, subclass, 0); + if (!class) + return 0; hlock->class_idx = class - lock_classes; curr->lockdep_depth = i; diff --git a/kernel/module/dups.c b/kernel/module/dups.c index 1d720a5311ba02..db737722970346 100644 --- a/kernel/module/dups.c +++ b/kernel/module/dups.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "internal.h" @@ -38,27 +39,37 @@ static bool enable_dups_trace = IS_ENABLED(CONFIG_MODULE_DEBUG_AUTOLOAD_DUPS_TRACE); module_param(enable_dups_trace, bool_enable_only, 0644); -/* - * Protects dup_kmod_reqs list, adds / removals with RCU. - */ +/* A mutex-protected list of active kmod requests. */ static DEFINE_MUTEX(kmod_dup_mutex); static LIST_HEAD(dup_kmod_reqs); struct kmod_dup_req { + refcount_t refcount; struct list_head list; char name[MODULE_NAME_LEN]; struct completion first_req_done; - struct work_struct complete_work; struct delayed_work delete_work; int dup_ret; }; +static void get_kmod_req(struct kmod_dup_req *kmod_req) +{ + refcount_inc(&kmod_req->refcount); +} + +static void put_kmod_req(struct kmod_dup_req *kmod_req) +{ + if (refcount_dec_and_test(&kmod_req->refcount)) + kfree(kmod_req); +} + static struct kmod_dup_req *kmod_dup_request_lookup(char *module_name) { struct kmod_dup_req *kmod_req; - list_for_each_entry_rcu(kmod_req, &dup_kmod_reqs, list, - lockdep_is_held(&kmod_dup_mutex)) { + lockdep_assert_held(&kmod_dup_mutex); + + list_for_each_entry(kmod_req, &dup_kmod_reqs, list) { if (strlen(kmod_req->name) == strlen(module_name) && !memcmp(kmod_req->name, module_name, strlen(module_name))) { return kmod_req; @@ -87,33 +98,10 @@ static void kmod_dup_request_delete(struct work_struct *work) * just returning 0. */ mutex_lock(&kmod_dup_mutex); - list_del_rcu(&kmod_req->list); - synchronize_rcu(); + list_del(&kmod_req->list); mutex_unlock(&kmod_dup_mutex); - kfree(kmod_req); -} - -static void kmod_dup_request_complete(struct work_struct *work) -{ - struct kmod_dup_req *kmod_req; - - kmod_req = container_of(work, struct kmod_dup_req, complete_work); - - /* - * This will ensure that the kernel will let all the waiters get - * informed its time to check the return value. It's time to - * go home. - */ - complete_all(&kmod_req->first_req_done); - /* - * Now that we have allowed prior request_module() calls to go on - * with life, let's schedule deleting this entry. We don't have - * to do it right away, but we *eventually* want to do it so to not - * let this linger forever as this is just a boot optimization for - * possible abuses of vmalloc() incurred by finit_module() thrashing. - */ - queue_delayed_work(system_dfl_wq, &kmod_req->delete_work, 60 * HZ); + put_kmod_req(kmod_req); } bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) @@ -129,8 +117,8 @@ bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) if (!new_kmod_req) return false; - memcpy(new_kmod_req->name, module_name, strlen(module_name)); - INIT_WORK(&new_kmod_req->complete_work, kmod_dup_request_complete); + refcount_set(&new_kmod_req->refcount, 1); + strscpy(new_kmod_req->name, module_name); INIT_DELAYED_WORK(&new_kmod_req->delete_work, kmod_dup_request_delete); init_completion(&new_kmod_req->first_req_done); @@ -161,10 +149,12 @@ bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) * keep tab on duplicates later. */ pr_debug("New request_module() for %s\n", module_name); - list_add_rcu(&new_kmod_req->list, &dup_kmod_reqs); + list_add(&new_kmod_req->list, &dup_kmod_reqs); mutex_unlock(&kmod_dup_mutex); return false; } + + get_kmod_req(kmod_req); mutex_unlock(&kmod_dup_mutex); /* We are dealing with a duplicate request now */ @@ -194,7 +184,7 @@ bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) * calls bail out right away. */ *dup_ret = 0; - return true; + goto out; } /* @@ -209,12 +199,14 @@ bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) TASK_KILLABLE); if (ret) { *dup_ret = ret; - return true; + goto out; } /* Now the duplicate request has the same exact return value as the first request */ *dup_ret = kmod_req->dup_ret; +out: + put_kmod_req(kmod_req); return true; } @@ -224,24 +216,31 @@ void kmod_dup_request_announce(char *module_name, int ret) mutex_lock(&kmod_dup_mutex); + /* + * Look for a kmod_dup_req previously added in + * kmod_dup_request_exists_wait(). Note that a request_module_nowait() + * without its own kmod_dup_req entry can announce a result of + * a concurrent request_module() call. + */ kmod_req = kmod_dup_request_lookup(module_name); - if (!kmod_req) - goto out; + if (!kmod_req || completion_done(&kmod_req->first_req_done)) { + mutex_unlock(&kmod_dup_mutex); + return; + } kmod_req->dup_ret = ret; - /* - * If we complete() here we may allow duplicate threads - * to continue before the first one that submitted the - * request. We're in no rush also, given that each and - * every bounce back to userspace is slow we avoid that - * with a slight delay here. So queueue up the completion - * and let duplicates suffer, just wait a tad bit longer. - * There is no rush. But we also don't want to hold the - * caller up forever or introduce any boot delays. - */ - queue_work(system_dfl_wq, &kmod_req->complete_work); + /* Inform all duplicate waiters to check the return value. */ + complete_all(&kmod_req->first_req_done); -out: mutex_unlock(&kmod_dup_mutex); + + /* + * Now that we have allowed prior request_module() calls to go on + * with life, let's schedule deleting this entry. We don't have + * to do it right away, but we *eventually* want to do it so to not + * let this linger forever as this is just a boot optimization for + * possible abuses of vmalloc() incurred by finit_module() thrashing. + */ + queue_delayed_work(system_dfl_wq, &kmod_req->delete_work, 60 * HZ); } diff --git a/kernel/module/stats.c b/kernel/module/stats.c index 3a9672f93a8ef7..78b59bc05387fa 100644 --- a/kernel/module/stats.c +++ b/kernel/module/stats.c @@ -253,7 +253,7 @@ int try_add_failed_module(const char *name, enum fail_dup_mod_reason reason) mod_fail = kzalloc_obj(*mod_fail); if (!mod_fail) return -ENOMEM; - memcpy(mod_fail->name, name, strlen(name)); + strscpy(mod_fail->name, name); __set_bit(reason, &mod_fail->dup_fail_mask); atomic_long_inc(&mod_fail->count); list_add_rcu(&mod_fail->list, &dup_failed_modules); diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index d933b5b2c05d45..4a73927cc55b6a 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -2797,9 +2797,10 @@ next: return error; error = memory_bm_create(&zero_bm, GFP_ATOMIC, PG_ANY); - if (error) + if (error) { + memory_bm_free(©_bm, PG_UNSAFE_CLEAR); return error; - + } nr_zero_pages = 0; hibernate_restore_protection_begin(); diff --git a/kernel/power/wakelock.c b/kernel/power/wakelock.c index fd763da06a8735..a8b6bd5ec46b49 100644 --- a/kernel/power/wakelock.c +++ b/kernel/power/wakelock.c @@ -63,7 +63,7 @@ static unsigned int number_of_wakelocks; static inline bool wakelocks_limit_exceeded(void) { - return number_of_wakelocks > CONFIG_PM_WAKELOCKS_LIMIT; + return number_of_wakelocks >= CONFIG_PM_WAKELOCKS_LIMIT; } static inline void increment_wakelocks_number(void) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 2fe9a963c823a4..6d363e42e2a057 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3264,10 +3264,8 @@ static bool console_flush_one_record(bool do_cond_resched, u64 *next_seq, bool * if (flags & CON_NBCON) { progress = nbcon_legacy_emit_next_record(con, handover, cookie, !do_cond_resched); - printk_seq = nbcon_seq_read(con); } else { progress = console_emit_next_record(con, handover, cookie); - printk_seq = con->seq; } /* @@ -3277,6 +3275,15 @@ static bool console_flush_one_record(bool do_cond_resched, u64 *next_seq, bool * if (*handover) goto fail; + /* + * @con can be used here now that it is certain that this + * context is still holding the SRCU read lock. + */ + if (flags & CON_NBCON) + printk_seq = nbcon_seq_read(con); + else + printk_seq = con->seq; + /* Track the next of the highest seq flushed. */ if (printk_seq > *next_seq) *next_seq = printk_seq; diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 95ad967adcf3cb..608f732866476f 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -970,7 +970,7 @@ void rcu_all_qs(void) { unsigned long flags; - if (!raw_cpu_read(rcu_data.rcu_urgent_qs)) + if (!READ_ONCE(*raw_cpu_ptr(&rcu_data.rcu_urgent_qs))) return; preempt_disable(); // For CONFIG_PREEMPT_COUNT=y kernels /* Load rcu_urgent_qs before other flags. */ @@ -978,8 +978,8 @@ void rcu_all_qs(void) preempt_enable(); return; } - this_cpu_write(rcu_data.rcu_urgent_qs, false); - if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) { + WRITE_ONCE(*this_cpu_ptr(&rcu_data.rcu_urgent_qs), false); + if (unlikely(READ_ONCE(*this_cpu_ptr(&rcu_data.rcu_need_heavy_qs)))) { local_irq_save(flags); rcu_momentary_eqs(); local_irq_restore(flags); @@ -999,8 +999,8 @@ void rcu_note_context_switch(bool preempt) /* Load rcu_urgent_qs before other flags. */ if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) goto out; - this_cpu_write(rcu_data.rcu_urgent_qs, false); - if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) + WRITE_ONCE(*this_cpu_ptr(&rcu_data.rcu_urgent_qs), false); + if (unlikely(READ_ONCE(*this_cpu_ptr(&rcu_data.rcu_need_heavy_qs)))) rcu_momentary_eqs(); out: rcu_tasks_qs(current, preempt); diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index 614ff0d33c0165..835df03e3d7b19 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -325,7 +325,7 @@ static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, u64 time, * A CPU running a task which woken up after an IO operation can have its * utilization boosted to speed up the completion of those IO operations. * The IO boost value is increased each time a task wakes up from IO, in - * sugov_iowait_apply(), and it's instead decreased by this function, + * sugov_iowait_boost(), and it's instead decreased by this function, * each time an increase has not been requested (!iowait_boost_pending). * * A CPU which also appears to have been idle for at least one tick has also diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 200300043fa54a..857dbe3519a868 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2734,15 +2734,17 @@ static int balance_dl(struct rq *rq, struct rq_flags *rf) */ static void wakeup_preempt_dl(struct rq *rq, struct task_struct *p, int flags) { + struct task_struct *donor = rq->donor; /* * Can only get preempted by stop-class, and those should be * few and short lived, doesn't really make sense to push * anything away for that. */ - if (p->sched_class != &dl_sched_class) + if (p->sched_class != &dl_sched_class || + donor->sched_class != &dl_sched_class) return; - if (dl_entity_preempt(&p->dl, &rq->donor->dl)) { + if (dl_entity_preempt(&p->dl, &donor->dl)) { resched_curr(rq); return; } diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index d7c4b62e712e5e..558be0cdebef8b 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -5291,6 +5291,7 @@ bool scx_allow_ttwu_queue(const struct task_struct *p) /** * handle_lockup - sched_ext common lockup handler + * @exit_cpu: CPU to record in exit_info. Pass the stalled/hung CPU, not current. * @fmt: format string * * Called on system stall or lockup condition and initiates abort of sched_ext @@ -5300,7 +5301,7 @@ bool scx_allow_ttwu_queue(const struct task_struct *p) * resolve the lockup. %false if sched_ext is not enabled or abort was already * initiated by someone else. */ -static __printf(1, 2) bool handle_lockup(const char *fmt, ...) +static __printf(2, 3) bool handle_lockup(int exit_cpu, const char *fmt, ...) { struct scx_sched *sch; va_list args; @@ -5316,7 +5317,7 @@ static __printf(1, 2) bool handle_lockup(const char *fmt, ...) case SCX_ENABLING: case SCX_ENABLED: va_start(args, fmt); - ret = scx_verror(sch, fmt, args); + ret = scx_vexit(sch, SCX_EXIT_ERROR, 0, exit_cpu, fmt, args); va_end(args); return ret; default: @@ -5338,7 +5339,7 @@ static __printf(1, 2) bool handle_lockup(const char *fmt, ...) */ bool scx_rcu_cpu_stall(void) { - return handle_lockup("RCU CPU stall detected!"); + return handle_lockup(-1, "RCU CPU stall detected!"); } /** @@ -5353,32 +5354,15 @@ bool scx_rcu_cpu_stall(void) */ void scx_softlockup(u32 dur_s) { - if (!handle_lockup("soft lockup - CPU %d stuck for %us", smp_processor_id(), dur_s)) + int cpu = smp_processor_id(); + + if (!handle_lockup(cpu, "soft lockup - CPU %d stuck for %us", cpu, dur_s)) return; printk_deferred(KERN_ERR "sched_ext: Soft lockup - CPU %d stuck for %us, disabling BPF scheduler\n", - smp_processor_id(), dur_s); -} - -/* - * scx_hardlockup() runs from NMI and eventually calls scx_claim_exit(), - * which takes scx_sched_lock. scx_sched_lock isn't NMI-safe and grabbing - * it from NMI context can lead to deadlocks. Defer via irq_work; the - * disable path runs off irq_work anyway. - */ -static atomic_t scx_hardlockup_cpu = ATOMIC_INIT(-1); - -static void scx_hardlockup_irq_workfn(struct irq_work *work) -{ - int cpu = atomic_xchg(&scx_hardlockup_cpu, -1); - - if (cpu >= 0 && handle_lockup("hard lockup - CPU %d", cpu)) - printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n", - cpu); + cpu, dur_s); } -static DEFINE_IRQ_WORK(scx_hardlockup_irq_work, scx_hardlockup_irq_workfn); - /** * scx_hardlockup - sched_ext hardlockup handler * @@ -5387,19 +5371,21 @@ static DEFINE_IRQ_WORK(scx_hardlockup_irq_work, scx_hardlockup_irq_workfn); * Try kicking out the current scheduler in an attempt to recover the system to * a good state before taking more drastic actions. * - * Queues an irq_work; the handle_lockup() call happens in IRQ context (see - * scx_hardlockup_irq_workfn). + * Called from NMI. Aborting the scheduler sets ->aborting throughout the + * hierarchy before returning, which is what breaks the dispatch-path live-locks + * that can hard-lock CPUs. * - * Returns %true if sched_ext is enabled and the work was queued, %false - * otherwise. + * Returns %true if sched_ext is enabled and abort was initiated, which may + * resolve the lockup. %false if sched_ext is not enabled or abort was already + * initiated by someone else. */ bool scx_hardlockup(int cpu) { - if (!rcu_access_pointer(scx_root)) + if (!handle_lockup(cpu, "hard lockup - CPU %d", cpu)) return false; - atomic_cmpxchg(&scx_hardlockup_cpu, -1, cpu); - irq_work_queue(&scx_hardlockup_irq_work); + printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n", + cpu); return true; } @@ -6308,12 +6294,9 @@ static void scx_root_disable(struct scx_sched *sch) percpu_up_write(&scx_fork_rwsem); /* - * Invalidate all the rq clocks to prevent getting outdated - * rq clocks from a previous scx scheduler. - * - * Also re-balance the dl_server bandwidth reservations: detach - * ext_server (no more sched_ext tasks) and reinstate fair_server if it - * was previously detached because we were running in full mode. + * Re-balance the dl_server bandwidth reservations: detach ext_server + * (no more sched_ext tasks) and reinstate fair_server if it was + * previously detached because we were running in full mode. * * Unlike the enable path, this runs on a recovery path that cannot * fail, so we use dl_server_swap_bw() to atomically free ext_server's @@ -6326,8 +6309,6 @@ static void scx_root_disable(struct scx_sched *sch) for_each_possible_cpu(cpu) { struct rq *rq = cpu_rq(cpu); - scx_rq_clock_invalidate(rq); - scoped_guard(rq_lock_irqsave, rq) { update_rq_clock(rq); if (was_switched_all) { diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 673059fa9d728b..b295991b9f10c5 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1544,8 +1544,6 @@ __printf(5, 6) bool __scx_exit(struct scx_sched *sch, enum scx_exit_kind kind, __scx_exit(sch, kind, exit_code, raw_smp_processor_id(), fmt, ##args) #define scx_error(sch, fmt, args...) \ scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) -#define scx_verror(sch, fmt, args) \ - scx_vexit((sch), SCX_EXIT_ERROR, 0, raw_smp_processor_id(), fmt, args) /* * Return the rq currently locked from an scx callback, or NULL if no rq is diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index f15f5764818ebb..4c7bb4b7018e28 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5174,7 +5174,8 @@ static inline void update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) { long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg; - u32 new_sum, divider; + u64 new_sum; + u32 divider; /* Nothing to update */ if (!delta_avg) @@ -5188,7 +5189,7 @@ update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cf /* Set new sched_entity's runnable */ se->avg.runnable_avg = gcfs_rq->avg.runnable_avg; - new_sum = se->avg.runnable_avg * divider; + new_sum = (u64)se->avg.runnable_avg * divider; delta_sum = (long)new_sum - (long)se->avg.runnable_sum; se->avg.runnable_sum = new_sum; @@ -9780,7 +9781,8 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f /* * XXX Getting preempted by higher class, try and find idle CPU? */ - if (p->sched_class != &fair_sched_class) + if (p->sched_class != &fair_sched_class || + donor->sched_class != &fair_sched_class) return; if (unlikely(se == pse)) @@ -11865,7 +11867,9 @@ static inline void update_sg_lb_stats(struct lb_env *env, /* Check for a misfit task on the cpu */ if (sgs->group_misfit_task_load < rq->misfit_task_load) { sgs->group_misfit_task_load = rq->misfit_task_load; - *sg_overloaded = 1; + + if (balancing_at_rd) + *sg_overloaded = 1; } } else if (env->idle && sched_reduced_capacity(rq, env->sd)) { /* Check for a task running on a CPU with reduced capacity */ @@ -11944,6 +11948,17 @@ static bool update_sd_pick_busiest(struct lb_env *env, sds->local_stat.group_type != group_has_spare)) return false; + /* + * Candidate sg has no more than one task per CPU and has higher + * per-CPU capacity. Migrating tasks to less capable CPUs may harm + * throughput. Maximize throughput, power/energy consequences are not + * considered. + */ + if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && + (sgs->group_type <= group_fully_busy) && + (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu)))) + return false; + if (sgs->group_type > busiest->group_type) return true; @@ -12050,17 +12065,6 @@ has_spare: break; } - /* - * Candidate sg has no more than one task per CPU and has higher - * per-CPU capacity. Migrating tasks to less capable CPUs may harm - * throughput. Maximize throughput, power/energy consequences are not - * considered. - */ - if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && - (sgs->group_type <= group_fully_busy) && - (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu)))) - return false; - return true; } diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index ef152d401fe205..156025ef81b75a 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -8,6 +8,7 @@ * */ #include +#include #include #include "sched.h" @@ -27,6 +28,7 @@ struct housekeeping { }; static struct housekeeping housekeeping; +static __initdata LLIST_HEAD(memblock_freelist); bool housekeeping_enabled(enum hk_type type) { @@ -189,10 +191,22 @@ void __init housekeeping_init(void) WARN_ON_ONCE(cpumask_empty(omask)); cpumask_copy(nmask, omask); RCU_INIT_POINTER(housekeeping.cpumasks[type], nmask); - memblock_free(omask, cpumask_size()); + __llist_add((struct llist_node *)omask, &memblock_freelist); } } +static int __init housekeeping_late_init(void) +{ + struct llist_node *llnode, *pos, *t; + + /* Free allocated memblock memory, if any */ + llnode = __llist_del_all(&memblock_freelist); + llist_for_each_safe(pos, t, llnode) + memblock_free(pos, cpumask_size()); + return 0; +} +pure_initcall(housekeeping_late_init); + static void __init housekeeping_setup_type(enum hk_type type, cpumask_var_t housekeeping_staging) { diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index e474c31d8fe66b..e6e5f8a2caafbd 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1629,7 +1629,8 @@ static void wakeup_preempt_rt(struct rq *rq, struct task_struct *p, int flags) /* * XXX If we're preempted by DL, queue a push? */ - if (p->sched_class != &rt_sched_class) + if (p->sched_class != &rt_sched_class || + donor->sched_class != &rt_sched_class) return; if (p->prio < donor->prio) { diff --git a/kernel/sys.c b/kernel/sys.c index df69bd71de032c..35b538ba843c70 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -2189,7 +2189,7 @@ static int prctl_set_auxv(struct mm_struct *mm, unsigned long addr, BUILD_BUG_ON(sizeof(user_auxv) != sizeof(mm->saved_auxv)); task_lock(current); - memcpy(mm->saved_auxv, user_auxv, len); + memcpy(mm->saved_auxv, user_auxv, sizeof(user_auxv)); task_unlock(current); return 0; diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index e48c4d379a7cee..5a786b3c778ce4 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -1566,8 +1566,12 @@ static int __init init_clocksource_sysfs(void) { int error = subsys_system_register(&clocksource_subsys, NULL); - if (!error) - error = device_register(&device_clocksource); + if (error) + return error; + + error = device_register(&device_clocksource); + if (error) + bus_unregister(&clocksource_subsys); return error; } diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 313dcea127fe48..96eef3026fbaeb 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -2228,8 +2228,10 @@ retry: expires_next = hrtimer_update_next_event(cpu_base); cpu_base->hang_detected = false; if (expires_next < now) { - if (++retries < 3) + if (++retries < 3) { + cpu_base->nr_retries++; goto retry; + } delta = ktime_sub(now, entry_time); cpu_base->max_hang_time = max_t(unsigned int, cpu_base->max_hang_time, delta); diff --git a/kernel/time/namespace.c b/kernel/time/namespace.c index 5fa0af66cf3f79..3aff27bb0a154f 100644 --- a/kernel/time/namespace.c +++ b/kernel/time/namespace.c @@ -293,10 +293,12 @@ int proc_timens_set_offset(struct file *file, struct task_struct *p, return -EINVAL; } - if (off->val.tv_sec > KTIME_SEC_MAX || - off->val.tv_sec < -KTIME_SEC_MAX) + if (off->val.tv_sec > KTIME_SEC_MAX || off->val.tv_sec < -KTIME_SEC_MAX) return -ERANGE; + if (off->val.tv_nsec < 0 || off->val.tv_nsec >= NSEC_PER_SEC) + return -EINVAL; + tp = timespec64_add(tp, off->val); /* * KTIME_SEC_MAX is divided by 2 to be sure that KTIME_MAX is diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 696bb119c56ef2..55ebf2703f1bfd 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -2390,6 +2390,11 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk, * xtime_nsec_2 = xtime_nsec_1 - offset * Which simplifies to: * xtime_nsec -= offset + * + * When subtracting offset from xtime_nsec, the same amount + * (in appropriate units) has to be added to ntp_error, in + * order to correctly track the delta between the time + * reported in xtime_nsec, and the intended time. */ if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) { /* NTP adjustment caused clocksource mult overflow */ @@ -2400,6 +2405,7 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk, tk->tkr_mono.mult += mult_adj; tk->xtime_interval += interval; tk->tkr_mono.xtime_nsec -= offset; + tk->ntp_error += offset << tk->ntp_error_shift; } /* @@ -3315,7 +3321,9 @@ static const struct attribute_group aux_clock_enable_attr_group = { static int __init tk_aux_sysfs_init(void) { struct kobject *auxo, *tko = kobject_create_and_add("time", kernel_kobj); + struct kobject *clks[MAX_AUX_CLOCKS]; int ret = -ENOMEM; + int i; if (!tko) return ret; @@ -3324,21 +3332,28 @@ static int __init tk_aux_sysfs_init(void) if (!auxo) goto err_clean; - for (int i = 0; i < MAX_AUX_CLOCKS; i++) { + for (i = 0; i < MAX_AUX_CLOCKS; i++) { char id[2] = { [0] = '0' + i, }; - struct kobject *clk = kobject_create_and_add(id, auxo); + clks[i] = kobject_create_and_add(id, auxo); - if (!clk) { + if (!clks[i]) { ret = -ENOMEM; - goto err_clean; + goto err_clks; } - ret = sysfs_create_group(clk, &aux_clock_enable_attr_group); + ret = sysfs_create_group(clks[i], &aux_clock_enable_attr_group); if (ret) - goto err_clean; + goto err_clk; } return 0; +err_clk: + kobject_put(clks[i]); +err_clks: + while (--i >= 0) { + sysfs_remove_group(clks[i], &aux_clock_enable_attr_group); + kobject_put(clks[i]); + } err_clean: kobject_put(auxo); kobject_put(tko); diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c index 806c23cf71fc94..059d43355e650f 100644 --- a/kernel/time/timer_migration.c +++ b/kernel/time/timer_migration.c @@ -1847,8 +1847,10 @@ static int tmigr_setup_groups(struct tmigr_hierarchy *hier, unsigned int cpu, } /* Assert single root without parent */ - if (WARN_ON_ONCE(i >= tmigr_hierarchy_levels)) + if (WARN_ON_ONCE(i >= tmigr_hierarchy_levels)) { + kfree(stack); return -EINVAL; + } for (; i >= start_lvl; i--) { group = stack[i]; diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index eb01f1d6506665..1925ee9cc36da5 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -572,7 +572,6 @@ struct ring_buffer_per_cpu { struct trace_buffer { unsigned flags; - int cpus; atomic_t record_disabled; atomic_t resizing; cpumask_var_t cpumask; @@ -2783,7 +2782,6 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, init_irq_work(&buffer->irq_work.work, rb_wake_up_waiters); init_waitqueue_head(&buffer->irq_work.waiters); - buffer->cpus = nr_cpu_ids; bsize = sizeof(void *) * nr_cpu_ids; buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()), diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index a6d5d1786469cf..fc964001783f3c 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -4669,7 +4669,7 @@ trace_event_update_with_eval_map(struct module *mod, map = start; - trace_event_update_all(map, len); + trace_event_update_all(map, len, mod); if (len <= 0) return; diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 191a64c885daba..3c111ca88e32dd 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -2286,13 +2286,13 @@ static inline const char *get_syscall_name(int syscall) #ifdef CONFIG_EVENT_TRACING void trace_event_init(void); -void trace_event_update_all(struct trace_eval_map **map, int len); +void trace_event_update_all(struct trace_eval_map **map, int len, struct module *mod); /* Used from boot time tracer */ extern int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set); extern int trigger_process_regex(struct trace_event_file *file, char *buff); #else static inline void __init trace_event_init(void) { } -static inline void trace_event_update_all(struct trace_eval_map **map, int len) { } +static inline void trace_event_update_all(struct trace_eval_map **map, int len, struct module *mod) { } #endif #ifdef CONFIG_TRACER_SNAPSHOT diff --git a/kernel/trace/trace_btf.c b/kernel/trace/trace_btf.c index d3ba356d5503e6..ee7a04886bf673 100644 --- a/kernel/trace/trace_btf.c +++ b/kernel/trace/trace_btf.c @@ -61,16 +61,17 @@ struct btf_anon_stack { /* * Find a member of data structure/union by name and return it. - * Return NULL if not found, or -EINVAL if parameter is invalid. - * If the member is an member of anonymous union/structure, the offset - * of that anonymous union/structure is stored into @anon_offset. Caller - * can calculate the correct offset from the root data structure by - * adding anon_offset to the member's offset. + * Return NULL if not found, or ERR_PTR(-EINVAL) if parameter is invalid. + * If the member is a member of an anonymous union/structure, the bit offset + * of that anonymous union/structure is stored into @anon_offset. + * If @member_type is non-NULL, the actual containing structure/union type + * of the found member is stored into @member_type. */ const struct btf_member *btf_find_struct_member(struct btf *btf, const struct btf_type *type, const char *member_name, - u32 *anon_offset) + u32 *anon_offset, + const struct btf_type **member_type) { struct btf_anon_stack *anon_stack; const struct btf_member *member; @@ -94,14 +95,16 @@ retry: if (mtype && btf_type_is_struct(mtype) && top < BTF_ANON_STACK_MAX) { anon_stack[top].tid = tid; - anon_stack[top++].offset = - cur_offset + member->offset; + anon_stack[top++].offset = cur_offset + + __btf_member_bit_offset(type, member); } } else { name = btf_name_by_offset(btf, member->name_off); if (name && !strcmp(member_name, name)) { if (anon_offset) *anon_offset = cur_offset; + if (member_type) + *member_type = type; goto out; } } diff --git a/kernel/trace/trace_btf.h b/kernel/trace/trace_btf.h index 4bc44bc261e6f9..4bd26bceae23aa 100644 --- a/kernel/trace/trace_btf.h +++ b/kernel/trace/trace_btf.h @@ -8,4 +8,5 @@ const struct btf_param *btf_get_func_param(const struct btf_type *func_proto, const struct btf_member *btf_find_struct_member(struct btf *btf, const struct btf_type *type, const char *member_name, - u32 *anon_offset); + u32 *anon_offset, + const struct btf_type **member_type); diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 640529409021f5..e03471109e4874 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -3577,7 +3577,7 @@ static void update_event_fields(struct trace_event_call *call, } /* Update all events for replacing eval and sanitizing */ -void trace_event_update_all(struct trace_eval_map **map, int len) +void trace_event_update_all(struct trace_eval_map **map, int len, struct module *mod) { struct trace_event_call *call, *p; const char *last_system = NULL; @@ -3589,6 +3589,10 @@ void trace_event_update_all(struct trace_eval_map **map, int len) mutex_lock(&event_mutex); down_write(&trace_event_sem); list_for_each_entry_safe(call, p, &ftrace_events, list) { + + if (mod && call->module != mod) + continue; + /* events are usually grouped together with systems */ if (!last_system || call->class->system != last_system) { first = true; diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index c8fd9b946f442e..aae60fbebeafbb 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -356,9 +356,8 @@ static bool btf_type_is_char_ptr(struct btf *btf, const struct btf_type *type) { const struct btf_type *real_type; u32 intdata; - s32 tid; - real_type = btf_type_skip_modifiers(btf, type->type, &tid); + real_type = btf_type_skip_modifiers(btf, type->type, NULL); if (!real_type) return false; @@ -375,14 +374,13 @@ static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type) const struct btf_type *real_type; const struct btf_array *array; u32 intdata; - s32 tid; if (BTF_INFO_KIND(type->info) != BTF_KIND_ARRAY) return false; array = (const struct btf_array *)(type + 1); - real_type = btf_type_skip_modifiers(btf, array->type, &tid); + real_type = btf_type_skip_modifiers(btf, array->type, NULL); intdata = btf_type_int(real_type); return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED) @@ -580,12 +578,12 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, { struct fetch_insn *code = *pcode; const struct btf_member *field; + const struct btf_type *mtype; u32 bitoffs, anon_offs; bool is_struct = ctx->struct_btf != NULL; struct btf *btf = ctx_btf(ctx); char *next; int is_ptr; - s32 tid; do { if (!is_struct) { @@ -596,7 +594,7 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, } /* Convert a struct pointer type to a struct type */ - type = btf_type_skip_modifiers(btf, type->type, &tid); + type = btf_type_skip_modifiers(btf, type->type, NULL); if (!type) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); return -EINVAL; @@ -615,7 +613,7 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, anon_offs = 0; field = btf_find_struct_member(btf, type, fieldname, - &anon_offs); + &anon_offs, &mtype); if (IS_ERR(field)) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); return PTR_ERR(field); @@ -628,7 +626,7 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, bitoffs += anon_offs; /* Accumulate the bit-offsets of the dot-connected fields */ - if (btf_type_kflag(type)) { + if (btf_type_kflag(mtype)) { bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset); ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset); } else { @@ -636,7 +634,7 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, ctx->last_bitsize = 0; } - type = btf_type_skip_modifiers(btf, field->type, &tid); + type = btf_type_skip_modifiers(btf, field->type, NULL); if (!type) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); return -EINVAL; @@ -755,7 +753,7 @@ static int parse_btf_arg(char *varname, return -ENOENT; found: - type = btf_type_skip_modifiers(ctx->btf, tid, &tid); + type = btf_type_skip_modifiers(ctx->btf, tid, NULL); found_type: if (!type) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 78068ae8f28a6c..929c04a9581bd0 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -7689,13 +7689,31 @@ module_param_named(panic_on_stall_time, wq_panic_on_stall_time, uint, 0644); MODULE_PARM_DESC(panic_on_stall_time, "Panic if stall exceeds this many seconds (0=disabled)"); /* - * Show workers that might prevent the processing of pending work items. - * A busy worker that is not running on the CPU (e.g. sleeping in - * wait_event_idle() with PF_WQ_WORKER cleared) can stall the pool just as - * effectively as a CPU-bound one, so dump every in-flight worker. + * Report that a pool has no worker in running state, which is a sign that the + * pool may be stuck. Print pool info. Must be called with pool->lock held and + * inside a printk_deferred_enter/exit region. + */ +static void show_pool_no_running_worker(struct worker_pool *pool) +{ + lockdep_assert_held(&pool->lock); + + printk_deferred_enter(); + pr_info("pool %d: no worker in running state, cpu=%d is %s (nr_workers=%d nr_idle=%d)\n", + pool->id, pool->cpu, + idle_cpu(pool->cpu) ? "idle" : "busy", + pool->nr_workers, pool->nr_idle); + pr_info("The pool might have trouble waking an idle worker.\n"); + printk_deferred_exit(); +} + +/* + * Show running workers that might prevent the processing of pending work items. + * If no running worker is found, the pool may be stuck waiting for an idle + * worker to be woken, so report the pool state. */ static void show_cpu_pool_busy_workers(struct worker_pool *pool) { + bool found_running = false; struct worker *worker; unsigned long irq_flags; int bkt; @@ -7703,6 +7721,11 @@ static void show_cpu_pool_busy_workers(struct worker_pool *pool) raw_spin_lock_irqsave(&pool->lock, irq_flags); hash_for_each(pool->busy_hash, bkt, worker, hentry) { + /* Skip workers that are not actively running on the CPU. */ + if (!task_is_running(worker->task)) + continue; + + found_running = true; /* * Defer printing to avoid deadlocks in console * drivers that queue work while holding locks @@ -7716,6 +7739,13 @@ static void show_cpu_pool_busy_workers(struct worker_pool *pool) printk_deferred_exit(); } + /* + * If no running worker was found, the pool is likely stuck. Print pool + * state. + */ + if (!found_running) + show_pool_no_running_worker(pool); + raw_spin_unlock_irqrestore(&pool->lock, irq_flags); } diff --git a/lib/interval_tree_test.c b/lib/interval_tree_test.c index eba2d3e2898009..b0b07270ce7c3c 100644 --- a/lib/interval_tree_test.c +++ b/lib/interval_tree_test.c @@ -140,13 +140,13 @@ static int intersection_range_check(void) intxn1 = bitmap_alloc(nnodes, GFP_KERNEL); if (!intxn1) { - WARN_ON_ONCE("Failed to allocate intxn1\n"); + WARN_ONCE(1, "Failed to allocate intxn1\n"); return -ENOMEM; } intxn2 = bitmap_alloc(nnodes, GFP_KERNEL); if (!intxn2) { - WARN_ON_ONCE("Failed to allocate intxn2\n"); + WARN_ONCE(1, "Failed to allocate intxn2\n"); bitmap_free(intxn1); return -ENOMEM; } diff --git a/lib/maple_tree.c b/lib/maple_tree.c index e52876435b77f0..1a3815973add0a 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -3793,35 +3793,40 @@ int mas_alloc_cyclic(struct ma_state *mas, unsigned long *startp, void *entry, unsigned long range_lo, unsigned long range_hi, unsigned long *next, gfp_t gfp) { - unsigned long min = range_lo; - int ret = 0; - - range_lo = max(min, *next); - ret = mas_empty_area(mas, range_lo, range_hi, 1); - if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) { - mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED; - ret = 1; - } - if (ret < 0 && range_lo > min) { - mas_reset(mas); - ret = mas_empty_area(mas, min, range_hi, 1); - if (ret == 0) - ret = 1; - } - if (ret < 0) - return ret; + int ret; + unsigned long min; + min = range_lo; do { + range_lo = max(min, *next); + ret = mas_empty_area(mas, range_lo, range_hi, 1); + if (ret < 0 && range_lo > min) { + mas_reset(mas); + ret = mas_empty_area(mas, min, range_hi, 1); + if (ret == 0) + ret = 1; + } + if (ret < 0) + goto out; + mas_insert(mas, entry); } while (mas_nomem(mas, gfp)); - if (mas_is_err(mas)) - return xa_err(mas->node); + if (mas_is_err(mas)) { + ret = xa_err(mas->node); + goto out; + } + + if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) { + mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED; + ret = 1; + } *startp = mas->index; *next = *startp + 1; if (*next == 0) mas->tree->ma_flags |= MT_FLAGS_ALLOC_WRAPPED; +out: mas_destroy(mas); return ret; } diff --git a/lib/string.c b/lib/string.c index a4e8ad23577d88..a3778d5aab4afd 100644 --- a/lib/string.c +++ b/lib/string.c @@ -821,7 +821,8 @@ void *memchr_inv(const void *start, int c, size_t bytes) { u8 value = c; u64 value64; - unsigned int words, prefix; + size_t words; + unsigned int prefix; if (bytes <= 16) return check_bytes8(start, value, bytes); diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c index 69813c10e6c0b9..448c3eb48a4a89 100644 --- a/lib/test_bitmap.c +++ b/lib/test_bitmap.c @@ -392,6 +392,7 @@ static void __init test_bitmap_sg(void) /* Scatter/gather relationship */ bitmap_zero(bmap_tmp, 100); + bitmap_zero(bmap_res, 100); bitmap_gather(bmap_tmp, bmap_scatter, sg_mask, nbits); bitmap_scatter(bmap_res, bmap_tmp, sg_mask, nbits); expect_eq_bitmap(bmap_scatter, bmap_res, 100); diff --git a/lib/test_hmm.c b/lib/test_hmm.c index c4adbf98fac790..45c0cb9922184a 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -407,7 +407,7 @@ static int dmirror_fault(struct dmirror *dmirror, unsigned long start, /* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) - return 0; + return -EFAULT; for (addr = start; addr < end; addr = range.end) { range.start = addr; diff --git a/mm/damon/core.c b/mm/damon/core.c index a86812d457c14a..372ca1161c57d3 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3118,18 +3118,20 @@ static void damon_verify_merge_regions_of(struct damon_region *r) * sz_limit size upper limit of each region */ static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, - unsigned long sz_limit) + unsigned long sz_limit, bool count_age) { struct damon_region *r, *prev = NULL, *next; damon_for_each_region_safe(r, next, t) { damon_verify_merge_regions_of(r); - if (abs(r->nr_accesses - r->last_nr_accesses) > thres) - r->age = 0; - else if ((r->nr_accesses == 0) != (r->last_nr_accesses == 0)) - r->age = 0; - else - r->age++; + if (count_age) { + if (abs(r->nr_accesses - r->last_nr_accesses) > thres) + r->age = 0; + else if ((r->nr_accesses == 0) != (r->last_nr_accesses == 0)) + r->age = 0; + else + r->age++; + } if (prev && prev->ar.end == r->ar.start && abs(prev->nr_accesses - r->nr_accesses) <= thres && @@ -3163,15 +3165,18 @@ static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, struct damon_target *t; unsigned int nr_regions; unsigned int max_thres; + bool count_age = true; max_thres = c->attrs.aggr_interval / (c->attrs.sample_interval ? c->attrs.sample_interval : 1); while (true) { nr_regions = 0; damon_for_each_target(t, c) { - damon_merge_regions_of(t, threshold, sz_limit); + damon_merge_regions_of(t, threshold, sz_limit, + count_age); nr_regions += damon_nr_regions(t); } + count_age = false; if (nr_regions <= c->attrs.max_nr_regions || max_thres <= threshold) break; diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index fbcc882dccc94b..e543cfff15909e 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -261,7 +261,7 @@ static void damon_test_merge_regions_of(struct kunit *test) damon_add_region(r, t); } - damon_merge_regions_of(t, 9, 9999); + damon_merge_regions_of(t, 9, 9999, true); /* 0-112, 114-130, 130-156, 156-170, 170-230, 230-10170 */ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 6u); if (damon_nr_regions(t) != 6) diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 7ac19fab226324..8b137328dcf014 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1263,7 +1263,8 @@ static pg_data_t *hotadd_init_pgdat(int nid) pgdat = NODE_DATA(nid); /* init node's zones as empty zones, we don't have any present pages.*/ - free_area_init_core_hotplug(pgdat); + if (free_area_init_core_hotplug(pgdat)) + return NULL; /* * The node we allocated has no zone fallback lists. For avoiding diff --git a/mm/mm_init.c b/mm/mm_init.c index ab428508186925..d50a54dbb1064b 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1338,7 +1338,7 @@ static void __init calculate_node_totalpages(struct pglist_data *pgdat, unsigned long node_start_pfn, unsigned long node_end_pfn) { - unsigned long realtotalpages = 0, totalpages = 0; + unsigned long realtotalpages = 0; enum zone_type i; for (i = 0; i < MAX_NR_ZONES; i++) { @@ -1368,11 +1368,10 @@ static void __init calculate_node_totalpages(struct pglist_data *pgdat, zone->present_early_pages = real_size; #endif - totalpages += spanned; realtotalpages += real_size; } - pgdat->node_spanned_pages = totalpages; + pgdat->node_spanned_pages = node_end_pfn - node_start_pfn; pgdat->node_present_pages = realtotalpages; pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages); } @@ -1536,7 +1535,7 @@ void __init set_pageblock_order(void) * NOTE: this function is only called during memory hotplug */ #ifdef CONFIG_MEMORY_HOTPLUG -void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) +int __ref free_area_init_core_hotplug(struct pglist_data *pgdat) { int nid = pgdat->node_id; enum zone_type z; @@ -1544,8 +1543,14 @@ void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) pgdat_init_internals(pgdat); - if (pgdat->per_cpu_nodestats == &boot_nodestats) - pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat); + if (pgdat->per_cpu_nodestats == &boot_nodestats) { + struct per_cpu_nodestat __percpu *p; + + p = alloc_percpu(struct per_cpu_nodestat); + if (!p) + return -ENOMEM; + pgdat->per_cpu_nodestats = p; + } /* * Reset the nr_zones, order and highest_zoneidx before reuse. @@ -1583,6 +1588,8 @@ void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) zone->present_pages = 0; zone_init_internals(zone, z, nid, 0); } + + return 0; } #endif diff --git a/mm/mremap.c b/mm/mremap.c index e9c8b1d05832be..a7bd785e603f32 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -1321,18 +1321,29 @@ static void dontunmap_complete(struct vma_remap_struct *vrm, { unsigned long start = vrm->addr; unsigned long end = vrm->addr + vrm->old_len; - unsigned long old_start = vrm->vma->vm_start; - unsigned long old_end = vrm->vma->vm_end; + struct vm_area_struct *vma = vrm->vma; + unsigned long old_start = vma->vm_start; + unsigned long old_end = vma->vm_end; /* We always clear VM_LOCKED[ONFAULT] on the old VMA. */ - vm_flags_clear(vrm->vma, VM_LOCKED_MASK); + vm_flags_clear(vma, VM_LOCKED_MASK); /* * anon_vma links of the old vma is no longer needed after its page * table has been moved. */ - if (new_vma != vrm->vma && start == old_start && end == old_end) - unlink_anon_vmas(vrm->vma); + if (new_vma != vma && start == old_start && end == old_end) { + const pgoff_t pgoff_unfaulted = vma->vm_start >> PAGE_SHIFT; + + unlink_anon_vmas(vma); + /* + * The VMA is now unfaulted and it is an invariant that + * unfaulted anonymous VMAs have page offset equal to + * vma->vm_start >> PAGE_SHIFT. + */ + if (vma_is_anonymous(vma) && !vma->vm_file) + vma->vm_pgoff = pgoff_unfaulted; + } /* Because we won't unmap we don't need to touch locked_vm. */ } diff --git a/mm/swap.c b/mm/swap.c index 588f50d8f1a8c9..1a87e56bdcc52d 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -1153,7 +1153,16 @@ static void lruvec_reparent_lru(struct lruvec *child_lruvec, for_each_managed_zone_pgdat(zone, NODE_DATA(nid), zid, MAX_NR_ZONES - 1) { unsigned long size = mem_cgroup_get_zone_lru_size(child_lruvec, lru, zid); + if (!size) + continue; + + /* + * The folios are accounted to the parent from now on, so the + * size has to be moved, not just copied. Leaving it behind + * makes the dying child describe folios it no longer owns. + */ mem_cgroup_update_lru_size(parent_lruvec, lru, zid, size); + mem_cgroup_update_lru_size(child_lruvec, lru, zid, -(long)size); } } diff --git a/mm/vmscan.c b/mm/vmscan.c index ee3efc6213f49d..8781dbcdb99a6d 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -4554,7 +4554,12 @@ void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, for_each_managed_zone_pgdat(zone, NODE_DATA(nid), zid, MAX_NR_ZONES - 1) { unsigned long size = mem_cgroup_get_zone_lru_size(child_lruvec, lru, zid); + if (!size) + continue; + + /* Move the accounting, do not duplicate it. */ mem_cgroup_update_lru_size(parent_lruvec, lru, zid, size); + mem_cgroup_update_lru_size(child_lruvec, lru, zid, -(long)size); } } } diff --git a/net/batman-adv/bridge_loop_avoidance.c b/net/batman-adv/bridge_loop_avoidance.c index 7a31bc21bf87fd..87d55695384363 100644 --- a/net/batman-adv/bridge_loop_avoidance.c +++ b/net/batman-adv/bridge_loop_avoidance.c @@ -694,12 +694,14 @@ static void batadv_bla_add_claim(struct batadv_priv *bat_priv, struct batadv_bla_backbone_gw *old_backbone_gw; struct batadv_bla_claim *claim; struct batadv_bla_claim search_claim; - bool remove_crc = false; int hash_added; + u16 claim_crc; + bool changed; ether_addr_copy(search_claim.addr, mac); search_claim.vid = vid; claim = batadv_claim_hash_find(bat_priv, &search_claim); + claim_crc = crc16(0, mac, ETH_ALEN); /* create a new claim entry if it does not exist yet. */ if (!claim) { @@ -731,43 +733,56 @@ static void batadv_bla_add_claim(struct batadv_priv *bat_priv, kfree(claim); return; } + + spin_lock_bh(&backbone_gw->crc_lock); + backbone_gw->crc ^= claim_crc; + spin_unlock_bh(&backbone_gw->crc_lock); + + WRITE_ONCE(backbone_gw->lasttime, jiffies); + + batadv_claim_put(claim); + return; + } + + WRITE_ONCE(claim->lasttime, jiffies); + + /* replace backbone_gw atomically and adjust reference counters */ + spin_lock_bh(&claim->backbone_lock); + if (claim->backbone_gw != backbone_gw) { + changed = true; + + old_backbone_gw = claim->backbone_gw; + kref_get(&backbone_gw->refcount); + claim->backbone_gw = backbone_gw; } else { - WRITE_ONCE(claim->lasttime, jiffies); - if (claim->backbone_gw == backbone_gw) - /* no need to register a new backbone */ - goto claim_free_ref; + old_backbone_gw = NULL; + changed = false; + } + spin_unlock_bh(&claim->backbone_lock); + if (changed) { batadv_dbg(BATADV_DBG_BLA, bat_priv, "%s(): changing ownership for %pM, vid %d to gw %pM\n", __func__, mac, batadv_print_vid(vid), backbone_gw->orig); - remove_crc = true; - } + /* add claim address to new backbone_gw */ + spin_lock_bh(&backbone_gw->crc_lock); + backbone_gw->crc ^= claim_crc; + spin_unlock_bh(&backbone_gw->crc_lock); - /* replace backbone_gw atomically and adjust reference counters */ - spin_lock_bh(&claim->backbone_lock); - old_backbone_gw = claim->backbone_gw; - kref_get(&backbone_gw->refcount); - claim->backbone_gw = backbone_gw; - spin_unlock_bh(&claim->backbone_lock); + WRITE_ONCE(backbone_gw->lasttime, jiffies); + } - if (remove_crc) { + if (old_backbone_gw) { /* remove claim address from old backbone_gw */ spin_lock_bh(&old_backbone_gw->crc_lock); - old_backbone_gw->crc ^= crc16(0, claim->addr, ETH_ALEN); + old_backbone_gw->crc ^= claim_crc; spin_unlock_bh(&old_backbone_gw->crc_lock); - } - - batadv_backbone_gw_put(old_backbone_gw); - /* add claim address to new backbone_gw */ - spin_lock_bh(&backbone_gw->crc_lock); - backbone_gw->crc ^= crc16(0, claim->addr, ETH_ALEN); - spin_unlock_bh(&backbone_gw->crc_lock); - WRITE_ONCE(backbone_gw->lasttime, jiffies); + batadv_backbone_gw_put(old_backbone_gw); + } -claim_free_ref: batadv_claim_put(claim); } diff --git a/net/batman-adv/distributed-arp-table.c b/net/batman-adv/distributed-arp-table.c index 6ca946da927583..95d8e3a3839025 100644 --- a/net/batman-adv/distributed-arp-table.c +++ b/net/batman-adv/distributed-arp-table.c @@ -376,18 +376,19 @@ batadv_dat_entry_hash_find(struct batadv_priv *bat_priv, __be32 ip, static void batadv_dat_entry_add(struct batadv_priv *bat_priv, __be32 ip, u8 *mac_addr, unsigned short vid) { + u64 u64_mac = ether_addr_to_u64(mac_addr); struct batadv_dat_entry *dat_entry; int hash_added; dat_entry = batadv_dat_entry_hash_find(bat_priv, ip, vid); /* if this entry is already known, just update it */ if (dat_entry) { - if (!batadv_compare_eth(dat_entry->mac_addr, mac_addr)) - ether_addr_copy(dat_entry->mac_addr, mac_addr); + atomic64_set(&dat_entry->mac_addr, u64_mac); + dat_entry->last_update = jiffies; batadv_dbg(BATADV_DBG_DAT, bat_priv, "Entry updated: %pI4 %pM (vid: %d)\n", - &dat_entry->ip, dat_entry->mac_addr, + &dat_entry->ip, mac_addr, batadv_print_vid(vid)); goto out; } @@ -398,7 +399,7 @@ static void batadv_dat_entry_add(struct batadv_priv *bat_priv, __be32 ip, dat_entry->ip = ip; dat_entry->vid = vid; - ether_addr_copy(dat_entry->mac_addr, mac_addr); + atomic64_set(&dat_entry->mac_addr, u64_mac); dat_entry->last_update = jiffies; kref_init(&dat_entry->refcount); @@ -414,7 +415,7 @@ static void batadv_dat_entry_add(struct batadv_priv *bat_priv, __be32 ip, } batadv_dbg(BATADV_DBG_DAT, bat_priv, "New entry added: %pI4 %pM (vid: %d)\n", - &dat_entry->ip, dat_entry->mac_addr, batadv_print_vid(vid)); + &dat_entry->ip, mac_addr, batadv_print_vid(vid)); out: batadv_dat_entry_put(dat_entry); @@ -868,6 +869,8 @@ batadv_dat_cache_dump_entry(struct sk_buff *msg, u32 portid, struct netlink_callback *cb, struct batadv_dat_entry *dat_entry) { + u8 mac[ETH_ALEN]; + u64 u64_mac; int msecs; void *hdr; @@ -880,11 +883,12 @@ batadv_dat_cache_dump_entry(struct sk_buff *msg, u32 portid, genl_dump_check_consistent(cb, hdr); msecs = jiffies_to_msecs(jiffies - dat_entry->last_update); + u64_mac = atomic64_read(&dat_entry->mac_addr); + u64_to_ether_addr(u64_mac, mac); if (nla_put_in_addr(msg, BATADV_ATTR_DAT_CACHE_IP4ADDRESS, dat_entry->ip) || - nla_put(msg, BATADV_ATTR_DAT_CACHE_HWADDRESS, ETH_ALEN, - dat_entry->mac_addr) || + nla_put(msg, BATADV_ATTR_DAT_CACHE_HWADDRESS, ETH_ALEN, mac) || nla_put_u16(msg, BATADV_ATTR_DAT_CACHE_VID, dat_entry->vid) || nla_put_u32(msg, BATADV_ATTR_LAST_SEEN_MSECS, msecs)) { genlmsg_cancel(msg, hdr); @@ -1151,6 +1155,8 @@ bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv, struct net_device *mesh_iface = bat_priv->mesh_iface; int hdr_size = 0; unsigned short vid; + u8 mac[ETH_ALEN]; + u64 u64_mac; if (!READ_ONCE(bat_priv->distributed_arp_table)) goto out; @@ -1178,6 +1184,9 @@ bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv, dat_entry = batadv_dat_entry_hash_find(bat_priv, ip_dst, vid); if (dat_entry) { + u64_mac = atomic64_read(&dat_entry->mac_addr); + u64_to_ether_addr(u64_mac, mac); + /* If the ARP request is destined for a local client the local * client will answer itself. DAT would only generate a * duplicate packet. @@ -1186,7 +1195,7 @@ bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv, * additional DAT answer may trigger kernel warnings about * a packet coming from the wrong port. */ - if (batadv_is_my_client(bat_priv, dat_entry->mac_addr, vid)) { + if (batadv_is_my_client(bat_priv, mac, vid)) { ret = true; goto out; } @@ -1196,18 +1205,16 @@ bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv, * the backbone gws belonging to our backbone has claimed the * destination. */ - if (!batadv_bla_check_claim(bat_priv, - dat_entry->mac_addr, vid)) { + if (!batadv_bla_check_claim(bat_priv, mac, vid)) { batadv_dbg(BATADV_DBG_DAT, bat_priv, "Device %pM claimed by another backbone gw. Don't send ARP reply!", - dat_entry->mac_addr); + mac); ret = true; goto out; } skb_new = batadv_dat_arp_create_reply(bat_priv, ip_dst, ip_src, - dat_entry->mac_addr, - hw_src, vid); + mac, hw_src, vid); if (!skb_new) goto out; @@ -1249,6 +1256,8 @@ bool batadv_dat_snoop_incoming_arp_request(struct batadv_priv *bat_priv, struct batadv_dat_entry *dat_entry = NULL; bool ret = false; unsigned short vid; + u8 mac[ETH_ALEN]; + u64 u64_mac; int err; if (!READ_ONCE(bat_priv->distributed_arp_table)) @@ -1276,8 +1285,11 @@ bool batadv_dat_snoop_incoming_arp_request(struct batadv_priv *bat_priv, if (!dat_entry) goto out; + u64_mac = atomic64_read(&dat_entry->mac_addr); + u64_to_ether_addr(u64_mac, mac); + skb_new = batadv_dat_arp_create_reply(bat_priv, ip_dst, ip_src, - dat_entry->mac_addr, hw_src, vid); + mac, hw_src, vid); if (!skb_new) goto out; @@ -1368,6 +1380,8 @@ bool batadv_dat_snoop_incoming_arp_reply(struct batadv_priv *bat_priv, u8 *hw_src, *hw_dst; bool dropped = false; unsigned short vid; + u8 mac[ETH_ALEN]; + u64 u64_mac; if (!READ_ONCE(bat_priv->distributed_arp_table)) goto out; @@ -1396,11 +1410,17 @@ bool batadv_dat_snoop_incoming_arp_reply(struct batadv_priv *bat_priv, * this frame would lead to doubled receive of an ARP reply. */ dat_entry = batadv_dat_entry_hash_find(bat_priv, ip_src, vid); - if (dat_entry && batadv_compare_eth(hw_src, dat_entry->mac_addr)) { - batadv_dbg(BATADV_DBG_DAT, bat_priv, "Doubled ARP reply removed: ARP MSG = [src: %pM-%pI4 dst: %pM-%pI4]; dat_entry: %pM-%pI4\n", - hw_src, &ip_src, hw_dst, &ip_dst, - dat_entry->mac_addr, &dat_entry->ip); - dropped = true; + if (dat_entry) { + u64_mac = atomic64_read(&dat_entry->mac_addr); + u64_to_ether_addr(u64_mac, mac); + + if (batadv_compare_eth(hw_src, mac)) { + batadv_dbg(BATADV_DBG_DAT, bat_priv, + "Doubled ARP reply removed: ARP MSG = [src: %pM-%pI4 dst: %pM-%pI4]; dat_entry: %pM-%pI4\n", + hw_src, &ip_src, hw_dst, &ip_dst, + mac, &dat_entry->ip); + dropped = true; + } } /* Update our internal cache with both the IP addresses the node got diff --git a/net/batman-adv/types.h b/net/batman-adv/types.h index b1f9f8964c3fd2..9ab0dba82f509b 100644 --- a/net/batman-adv/types.h +++ b/net/batman-adv/types.h @@ -2176,7 +2176,7 @@ struct batadv_dat_entry { __be32 ip; /** @mac_addr: the MAC address associated to the stored IPv4 */ - u8 mac_addr[ETH_ALEN]; + atomic64_t mac_addr; /** @vid: the vlan ID associated to this entry */ unsigned short vid; diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index 7e4d737a795f9b..8de98af2fb5818 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -283,8 +283,6 @@ static int hci_enhanced_setup_sync(struct hci_dev *hdev, void *data) struct hci_cp_enhanced_setup_sync_conn cp; const struct sco_param *param; - kfree(conn_handle); - if (!hci_conn_valid(hdev, conn)) return -ECANCELED; @@ -453,6 +451,15 @@ static bool hci_setup_sync_conn(struct hci_conn *conn, __u16 handle) return true; } +static void hci_enhanced_setup_sync_destroy(struct hci_dev *hdev, void *data, + int err) +{ + struct conn_handle_t *conn_handle = data; + + hci_conn_put(conn_handle->conn); + kfree(conn_handle); +} + bool hci_setup_sync(struct hci_conn *conn, __u16 handle) { int result; @@ -464,12 +471,15 @@ bool hci_setup_sync(struct hci_conn *conn, __u16 handle) if (!conn_handle) return false; - conn_handle->conn = conn; + conn_handle->conn = hci_conn_get(conn); conn_handle->handle = handle; result = hci_cmd_sync_queue(conn->hdev, hci_enhanced_setup_sync, - conn_handle, NULL); - if (result < 0) + conn_handle, + hci_enhanced_setup_sync_destroy); + if (result < 0) { + hci_conn_put(conn); kfree(conn_handle); + } return result == 0; } diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index f44f02ddb94e54..a8d2a3f076e983 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -540,8 +540,6 @@ static int adv_timeout_expire_sync(struct hci_dev *hdev, void *data) { u8 instance = *(u8 *)data; - kfree(data); - hci_clear_adv_instance_sync(hdev, NULL, instance, false); if (list_empty(&hdev->adv_instances)) @@ -550,6 +548,12 @@ static int adv_timeout_expire_sync(struct hci_dev *hdev, void *data) return 0; } +static void adv_timeout_expire_destroy(struct hci_dev *hdev, void *data, + int err) +{ + kfree(data); +} + static void adv_timeout_expire(struct work_struct *work) { u8 *inst_ptr; @@ -570,7 +574,9 @@ static void adv_timeout_expire(struct work_struct *work) goto unlock; *inst_ptr = hdev->cur_adv_instance; - hci_cmd_sync_queue(hdev, adv_timeout_expire_sync, inst_ptr, NULL); + if (hci_cmd_sync_queue(hdev, adv_timeout_expire_sync, inst_ptr, + adv_timeout_expire_destroy) < 0) + kfree(inst_ptr); unlock: hci_dev_unlock(hdev); @@ -1281,6 +1287,7 @@ hci_set_ext_adv_params_sync(struct hci_dev *hdev, u8 instance, } static int hci_set_ext_adv_data_sync(struct hci_dev *hdev, u8 instance) + __context_unsafe(/* conditional locking */) { DEFINE_FLEX(struct hci_cp_le_set_ext_adv_data, pdu, data, length, HCI_MAX_EXT_AD_LENGTH); @@ -1369,6 +1376,7 @@ int hci_update_adv_data_sync(struct hci_dev *hdev, u8 instance) } int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) + __context_unsafe(/* conditional locking */) { struct hci_cp_le_set_ext_adv_params cp; struct hci_rp_le_set_ext_adv_params rp; @@ -1525,6 +1533,7 @@ int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) } static int hci_set_ext_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) + __context_unsafe(/* conditional locking */) { DEFINE_FLEX(struct hci_cp_le_set_ext_scan_rsp_data, pdu, data, length, HCI_MAX_EXT_AD_LENGTH); @@ -1578,6 +1587,7 @@ static int hci_set_ext_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) } static int __hci_set_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) + __context_unsafe(/* conditional locking */) { struct hci_cp_le_set_scan_rsp_data cp; u8 len; @@ -1719,6 +1729,7 @@ static int hci_set_per_adv_params_sync(struct hci_dev *hdev, u8 instance, } static int hci_set_per_adv_data_sync(struct hci_dev *hdev, u8 instance) + __context_unsafe(/* conditional locking */) { DEFINE_FLEX(struct hci_cp_le_set_per_adv_data, pdu, data, length, HCI_MAX_PER_AD_LENGTH); @@ -7210,8 +7221,13 @@ static void create_le_conn_complete(struct hci_dev *hdev, void *data, int err) goto unlock; } - /* Check if connection is still pending */ - if (conn != hci_lookup_le_connect(hdev)) + /* Check if this connection is still pending. + * + * hci_lookup_le_connect() returns only the first LE connection + * in BT_CONNECT, which is not necessarily this one when two are + * pending at once, so ask the connection itself. + */ + if (conn->state != BT_CONNECT) goto unlock; /* Flush to make sure we send create conn cancel command if needed */ diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 1156aba4e83c4c..30d7120d3a150c 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -1833,7 +1833,10 @@ static void l2cap_conn_del(struct hci_conn *hcon, int err) hci_chan_del(conn->hchan); conn->hchan = NULL; + spin_lock(&hcon->proto_lock); hcon->l2cap_data = NULL; + spin_unlock(&hcon->proto_lock); + mutex_unlock(&conn->lock); l2cap_conn_put(conn); } @@ -7168,8 +7171,6 @@ static struct l2cap_conn *l2cap_conn_add(struct hci_conn *hcon) } kref_init(&conn->ref); - hcon->l2cap_data = conn; - conn->hcon = hci_conn_get(hcon); conn->hchan = hchan; BT_DBG("hcon %p conn %p hchan %p", hcon, conn, hchan); @@ -7198,6 +7199,11 @@ static struct l2cap_conn *l2cap_conn_add(struct hci_conn *hcon) conn->disc_reason = HCI_ERROR_REMOTE_USER_TERM; + spin_lock(&hcon->proto_lock); + conn->hcon = hci_conn_get(hcon); + hcon->l2cap_data = conn; + spin_unlock(&hcon->proto_lock); + return conn; } @@ -7582,13 +7588,18 @@ next: int l2cap_disconn_ind(struct hci_conn *hcon) { - struct l2cap_conn *conn = hcon->l2cap_data; + struct l2cap_conn *conn; + int ret = HCI_ERROR_REMOTE_USER_TERM; BT_DBG("hcon %p", hcon); - if (!conn) - return HCI_ERROR_REMOTE_USER_TERM; - return conn->disc_reason; + spin_lock(&hcon->proto_lock); + conn = hcon->l2cap_data; + if (conn) + ret = conn->disc_reason; + spin_unlock(&hcon->proto_lock); + + return ret; } static void l2cap_disconn_cfm(struct hci_conn *hcon, u8 reason) diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index 4058ff50cc27a8..143654a47989d3 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -436,11 +436,26 @@ static int l2cap_get_mode(struct l2cap_chan *chan) return -EINVAL; } +static struct l2cap_conn *l2cap_chan_conn(struct l2cap_chan *chan) +{ + lockdep_assert_held(&chan->lock); + + /* l2cap_conn_del() sets FLAG_DEL while holding chan->lock before + * conn->hcon is deleted. If not set and conn is non-NULL, conn->hcon + * remains alive during this chan->lock critical section. + */ + if (test_bit(FLAG_DEL, &chan->flags)) + return NULL; + + return chan->conn; +} + static int l2cap_sock_getsockopt_old(struct socket *sock, int optname, sockopt_t *sopt) { struct sock *sk = sock->sk; struct l2cap_chan *chan = l2cap_pi(sk)->chan; + struct l2cap_conn *conn; struct l2cap_options opts; struct l2cap_conninfo cinfo; int err = 0; @@ -451,6 +466,7 @@ static int l2cap_sock_getsockopt_old(struct socket *sock, int optname, len = sopt->optlen; + l2cap_chan_lock(chan); lock_sock(sk); switch (optname) { @@ -537,9 +553,15 @@ static int l2cap_sock_getsockopt_old(struct socket *sock, int optname, break; } + conn = l2cap_chan_conn(chan); + if (!conn) { + err = -ENOTCONN; + break; + } + memset(&cinfo, 0, sizeof(cinfo)); - cinfo.hci_handle = chan->conn->hcon->handle; - memcpy(cinfo.dev_class, chan->conn->hcon->dev_class, 3); + cinfo.hci_handle = conn->hcon->handle; + memcpy(cinfo.dev_class, conn->hcon->dev_class, 3); len = min(len, sizeof(cinfo)); if (copy_to_iter(&cinfo, len, &sopt->iter_out) != len) @@ -553,6 +575,8 @@ static int l2cap_sock_getsockopt_old(struct socket *sock, int optname, } release_sock(sk); + l2cap_chan_unlock(chan); + return err; } @@ -561,6 +585,7 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname, { struct sock *sk = sock->sk; struct l2cap_chan *chan = l2cap_pi(sk)->chan; + struct l2cap_conn *conn; struct bt_security sec; struct bt_power pwr; int len, mode, err = 0; @@ -578,6 +603,7 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname, len = sopt->optlen; + l2cap_chan_lock(chan); lock_sock(sk); switch (optname) { @@ -589,12 +615,14 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname, break; } + conn = l2cap_chan_conn(chan); + memset(&sec, 0, sizeof(sec)); - if (chan->conn) { - sec.level = chan->conn->hcon->sec_level; + if (conn) { + sec.level = conn->hcon->sec_level; if (sk->sk_state == BT_CONNECTED) - sec.key_size = chan->conn->hcon->enc_key_size; + sec.key_size = conn->hcon->enc_key_size; } else { sec.level = chan->sec_level; } @@ -678,12 +706,14 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname, break; case BT_PHY: - if (sk->sk_state != BT_CONNECTED) { + conn = l2cap_chan_conn(chan); + + if (sk->sk_state != BT_CONNECTED || !conn) { err = -ENOTCONN; break; } - opt = hci_conn_get_phy(chan->conn->hcon); + opt = hci_conn_get_phy(conn->hcon); if (copy_to_iter(&opt, sizeof(opt), &sopt->iter_out) != sizeof(opt)) @@ -719,6 +749,7 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname, } release_sock(sk); + l2cap_chan_unlock(chan); return err; } @@ -749,6 +780,7 @@ static int l2cap_sock_setsockopt_old(struct socket *sock, int optname, BT_DBG("sk %p", sk); + l2cap_chan_lock(chan); lock_sock(sk); switch (optname) { @@ -850,6 +882,7 @@ static int l2cap_sock_setsockopt_old(struct socket *sock, int optname, } release_sock(sk); + l2cap_chan_unlock(chan); return err; } @@ -913,6 +946,7 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, if (level != SOL_BLUETOOTH) return -ENOPROTOOPT; + l2cap_chan_lock(chan); lock_sock(sk); switch (optname) { @@ -938,11 +972,10 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, chan->sec_level = sec.level; - if (!chan->conn) + conn = l2cap_chan_conn(chan); + if (!conn) break; - conn = chan->conn; - /* change security for LE channels */ if (chan->scid == L2CAP_CID_ATT) { if (smp_conn_security(conn->hcon, sec.level)) { @@ -997,7 +1030,8 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, } if (opt == BT_FLUSHABLE_OFF) { - conn = chan->conn; + conn = l2cap_chan_conn(chan); + /* proceed further only when we have l2cap_conn and No Flush support in the LM */ if (!conn || !lmp_no_flush_capable(conn->hcon->hdev)) { @@ -1083,7 +1117,8 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, break; case BT_PHY: - if (sk->sk_state != BT_CONNECTED) { + conn = l2cap_chan_conn(chan); + if (sk->sk_state != BT_CONNECTED || !conn) { err = -ENOTCONN; break; } @@ -1093,10 +1128,6 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, if (err) break; - if (!chan->conn) - break; - - conn = chan->conn; err = hci_conn_set_phy(conn->hcon, phys); break; @@ -1139,6 +1170,7 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, } release_sock(sk); + l2cap_chan_unlock(chan); return err; } @@ -1312,7 +1344,12 @@ static void l2cap_sock_kill(struct sock *sk) BT_DBG("sk %p state %s", sk, state_to_string(sk->sk_state)); + /* Take lock to synchronize against access without owning sk->sk_socket, + * eg. in l2cap_sock_cleanup_listen(). proto_ops etc. don't need lock. + */ + lock_sock(sk); l2cap_sock_put_chan(sk); + release_sock(sk); /* Kill poor orphan */ sock_set_flag(sk, SOCK_DEAD); @@ -1515,14 +1552,10 @@ static void l2cap_sock_cleanup_listen(struct sock *parent) * establish sk_lock -> conn->lock and invert the established * conn->lock -> chan->lock -> sk_lock order (lockdep deadlock). * - * Instead, briefly take the child sk lock to fetch and pin its chan. - * l2cap_conn_del() reaches the chan free only via - * l2cap_chan_del() -> l2cap_sock_teardown_cb(), which itself takes - * the child sk lock; holding it across l2cap_chan_hold_unless_zero() - * therefore guarantees the chan cannot be freed while we read and - * pin it (hold_unless_zero() additionally skips a chan already past - * its last reference). We then drop the sk lock before taking - * chan->lock, so sk and chan locks are never held together. + * Instead, briefly take the child sk lock to synchronize vs. + * l2cap_sock_kill that puts l2cap_pi(sk)->chan. We then drop the sk + * lock before taking chan->lock, so sk and chan locks are never held + * together. * * Since we cannot call l2cap_chan_close() without conn->lock, * schedule l2cap_chan_timeout to close the channel; it already @@ -1532,10 +1565,12 @@ static void l2cap_sock_cleanup_listen(struct sock *parent) struct l2cap_chan *chan; lock_sock_nested(sk, L2CAP_NESTING_NORMAL); - chan = l2cap_chan_hold_unless_zero(l2cap_pi(sk)->chan); + chan = l2cap_pi(sk)->chan; + if (chan) + l2cap_chan_hold(chan); release_sock(sk); if (!chan) { - /* l2cap_conn_del() already tearing this child down */ + /* Already torn down */ sock_put(sk); continue; } @@ -1567,6 +1602,11 @@ static int l2cap_sock_new_connection_cb(struct l2cap_chan *chan, lock_sock(parent); + if (parent->sk_state != BT_LISTEN) { + release_sock(parent); + return -EINVAL; + } + /* Check for backlog size */ if (sk_acceptq_is_full(parent)) { BT_DBG("backlog full %d", parent->sk_ack_backlog); @@ -1730,10 +1770,14 @@ static void l2cap_sock_state_change_cb(struct l2cap_chan *chan, int state, if (!sk) return; + lock_sock(sk); + sk->sk_state = state; if (err) sk->sk_err = err; + + release_sock(sk); } static struct sk_buff *l2cap_sock_alloc_skb_cb(struct l2cap_chan *chan, @@ -1809,6 +1853,8 @@ static void l2cap_sock_resume_cb(struct l2cap_chan *chan) if (!sk) return; + lock_sock(sk); + if (test_and_clear_bit(FLAG_PENDING_SECURITY, &chan->flags)) { sk->sk_state = BT_CONNECTED; chan->state = BT_CONNECTED; @@ -1816,6 +1862,8 @@ static void l2cap_sock_resume_cb(struct l2cap_chan *chan) clear_bit(BT_SK_SUSPEND, &bt_sk(sk)->flags); sk->sk_state_change(sk); + + release_sock(sk); } static void l2cap_sock_set_shutdown_cb(struct l2cap_chan *chan) diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c index c16b0b80c1939e..deced8a5bc2d6c 100644 --- a/net/bluetooth/mgmt.c +++ b/net/bluetooth/mgmt.c @@ -2431,11 +2431,15 @@ static int send_cancel(struct hci_dev *hdev, void *data) mgmt_cmd_complete(cmd->sk, hdev->id, MGMT_OP_MESH_SEND_CANCEL, 0, NULL, 0); - mgmt_pending_free(cmd); return 0; } +static void send_cancel_destroy(struct hci_dev *hdev, void *data, int err) +{ + mgmt_pending_free(data); +} + static int mesh_send_cancel(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { @@ -2456,7 +2460,8 @@ static int mesh_send_cancel(struct sock *sk, struct hci_dev *hdev, if (!cmd) err = -ENOMEM; else - err = hci_cmd_sync_queue(hdev, send_cancel, cmd, NULL); + err = hci_cmd_sync_queue(hdev, send_cancel, cmd, + send_cancel_destroy); if (err < 0) { err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND_CANCEL, @@ -2642,7 +2647,7 @@ static int send_hci_cmd_sync(struct hci_dev *hdev, void *data) if (IS_ERR(skb)) { mgmt_cmd_status(cmd->sk, hdev->id, MGMT_OP_HCI_CMD_SYNC, mgmt_status(PTR_ERR(skb))); - goto done; + return 0; } mgmt_cmd_complete(cmd->sk, hdev->id, MGMT_OP_HCI_CMD_SYNC, 0, @@ -2650,12 +2655,14 @@ static int send_hci_cmd_sync(struct hci_dev *hdev, void *data) kfree_skb(skb); -done: - mgmt_pending_free(cmd); - return 0; } +static void send_hci_cmd_sync_destroy(struct hci_dev *hdev, void *data, int err) +{ + mgmt_pending_free(data); +} + static int mgmt_hci_cmd_sync(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { @@ -2681,7 +2688,8 @@ static int mgmt_hci_cmd_sync(struct sock *sk, struct hci_dev *hdev, if (!cmd) err = -ENOMEM; else - err = hci_cmd_sync_queue(hdev, send_hci_cmd_sync, cmd, NULL); + err = hci_cmd_sync_queue(hdev, send_hci_cmd_sync, cmd, + send_hci_cmd_sync_destroy); if (err < 0) { err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_HCI_CMD_SYNC, @@ -6155,6 +6163,7 @@ static int start_service_discovery(struct sock *sk, struct hci_dev *hdev, struct mgmt_pending_cmd *cmd; const u16 max_uuid_count = ((U16_MAX - sizeof(*cp)) / 16); u16 uuid_count, expected_len; + u8 (*uuids)[16] = NULL; u8 status; int err; @@ -6231,12 +6240,10 @@ static int start_service_discovery(struct sock *sk, struct hci_dev *hdev, hdev->discovery.result_filtering = true; hdev->discovery.type = cp->type; hdev->discovery.rssi = cp->rssi; - hdev->discovery.uuid_count = uuid_count; if (uuid_count > 0) { - hdev->discovery.uuids = kmemdup(cp->uuids, uuid_count * 16, - GFP_KERNEL); - if (!hdev->discovery.uuids) { + uuids = kmemdup(cp->uuids, uuid_count * sizeof(*uuids), GFP_KERNEL); + if (!uuids) { err = mgmt_cmd_complete(sk, hdev->id, MGMT_OP_START_SERVICE_DISCOVERY, MGMT_STATUS_FAILED, @@ -6246,6 +6253,11 @@ static int start_service_discovery(struct sock *sk, struct hci_dev *hdev, } } + spin_lock(&hdev->discovery.lock); + hdev->discovery.uuids = uuids; + hdev->discovery.uuid_count = uuid_count; + spin_unlock(&hdev->discovery.lock); + err = hci_cmd_sync_queue(hdev, start_discovery_sync, cmd, start_discovery_complete); if (err < 0) { @@ -10375,6 +10387,7 @@ static bool is_filter_match(struct hci_dev *hdev, s8 rssi, u8 *eir, !hci_test_quirk(hdev, HCI_QUIRK_STRICT_DUPLICATE_FILTER)))) return false; + spin_lock(&hdev->discovery.lock); if (hdev->discovery.uuid_count != 0) { /* If a list of UUIDs is provided in filter, results with no * matching UUID should be dropped. @@ -10383,9 +10396,12 @@ static bool is_filter_match(struct hci_dev *hdev, s8 rssi, u8 *eir, hdev->discovery.uuids) && !eir_has_uuids(scan_rsp, scan_rsp_len, hdev->discovery.uuid_count, - hdev->discovery.uuids)) + hdev->discovery.uuids)) { + spin_unlock(&hdev->discovery.lock); return false; + } } + spin_unlock(&hdev->discovery.lock); /* If duplicate filtering does not report RSSI changes, then restart * scanning to ensure updated result with updated RSSI values. diff --git a/net/bluetooth/msft.c b/net/bluetooth/msft.c index d7badce8746ca3..ded68568e6c9b5 100644 --- a/net/bluetooth/msft.c +++ b/net/bluetooth/msft.c @@ -165,6 +165,11 @@ static bool read_supported_features(struct hci_dev *hdev, if (rp->sub_opcode != MSFT_OP_READ_SUPPORTED_FEATURES) goto failed; + if (skb->len < sizeof(*rp) + rp->evt_prefix_len) { + bt_dev_err(hdev, "MSFT event prefix length mismatch"); + goto failed; + } + if (rp->evt_prefix_len > 0) { msft->evt_prefix = kmemdup(rp->evt_prefix, rp->evt_prefix_len, GFP_KERNEL); diff --git a/net/bluetooth/rfcomm/core.c b/net/bluetooth/rfcomm/core.c index 5d150e8623d54d..63fa0f542ccf1d 100644 --- a/net/bluetooth/rfcomm/core.c +++ b/net/bluetooth/rfcomm/core.c @@ -1455,6 +1455,10 @@ static int rfcomm_apply_pn(struct rfcomm_dlc *d, int cr, struct rfcomm_pn *pn) d->mtu = __le16_to_cpu(pn->mtu); + /* MTU 0 causes an infinite loop when fragmenting in sendmsg */ + if (!d->mtu) + d->mtu = RFCOMM_DEFAULT_MTU; + if (cr && d->mtu > s->mtu) d->mtu = s->mtu; diff --git a/net/bridge/br_arp_nd_proxy.c b/net/bridge/br_arp_nd_proxy.c index 23eb6931a2b4ad..fd65d6af4f35a0 100644 --- a/net/bridge/br_arp_nd_proxy.c +++ b/net/bridge/br_arp_nd_proxy.c @@ -195,13 +195,15 @@ void br_do_proxy_suppress_arp(struct sk_buff *skb, struct net_bridge *br, n = neigh_lookup(&arp_tbl, &tip, vlandev); if (n) { struct net_bridge_fdb_entry *f; + u8 ha[ETH_ALEN] __aligned(2); if (!(READ_ONCE(n->nud_state) & NUD_VALID)) { neigh_release(n); return; } - f = br_fdb_find_rcu(br, n->ha, vid); + neigh_ha_snapshot(ha, n, n->dev); + f = br_fdb_find_rcu(br, ha, vid); if (f) { const struct net_bridge_port *dst = READ_ONCE(f->dst); bool replied = false; @@ -211,10 +213,10 @@ void br_do_proxy_suppress_arp(struct sk_buff *skb, struct net_bridge *br, br_is_neigh_suppress_enabled(dst, vid)) { if (!vid) br_arp_send(br, p, skb->dev, sip, tip, - sha, n->ha, sha, 0, 0); + sha, ha, sha, 0, 0); else br_arp_send(br, p, skb->dev, sip, tip, - sha, n->ha, sha, + sha, ha, sha, skb->vlan_proto, skb_vlan_tag_get(skb)); replied = true; @@ -252,7 +254,7 @@ struct nd_msg *br_is_nd_neigh_msg(const struct sk_buff *skb, struct nd_msg *msg) } static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, - struct sk_buff *request, struct neighbour *n, + struct sk_buff *request, struct neighbour *n, u8 *ha, __be16 vlan_proto, u16 vlan_tci) { struct net_device *dev = request->dev; @@ -310,7 +312,7 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, ipv6_eth_mc_map(&in6addr_linklocal_allnodes, eth_hdr(reply)->h_dest); else ether_addr_copy(eth_hdr(reply)->h_dest, daddr); - ether_addr_copy(eth_hdr(reply)->h_source, n->ha); + ether_addr_copy(eth_hdr(reply)->h_source, ha); eth_hdr(reply)->h_proto = htons(ETH_P_IPV6); reply->protocol = htons(ETH_P_IPV6); @@ -340,7 +342,7 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, na->icmph.icmp6_override = 1; na->icmph.icmp6_solicited = dad ? 0 : 1; na->target = ns->target; - ether_addr_copy(&na->opt[2], n->ha); + ether_addr_copy(&na->opt[2], ha); na->opt[0] = ND_OPT_TARGET_LL_ADDR; na->opt[1] = na_olen >> 3; @@ -369,7 +371,7 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, __vlan_hwaccel_put_tag(reply, vlan_proto, vlan_tci); netdev_dbg(dev, "nd send dev %s dst %pI6 dst_hw %pM src %pI6 src_hw %pM\n", - dev->name, &pip6->daddr, daddr, &pip6->saddr, n->ha); + dev->name, &pip6->daddr, daddr, &pip6->saddr, ha); if (p) { dev_queue_xmit(reply); @@ -472,24 +474,26 @@ void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br, n = neigh_lookup(&nd_tbl, &msg->target, vlandev); if (n) { struct net_bridge_fdb_entry *f; + u8 ha[ETH_ALEN] __aligned(2); if (!(READ_ONCE(n->nud_state) & NUD_VALID)) { neigh_release(n); return; } - f = br_fdb_find_rcu(br, n->ha, vid); + neigh_ha_snapshot(ha, n, n->dev); + f = br_fdb_find_rcu(br, ha, vid); if (f) { const struct net_bridge_port *dst = READ_ONCE(f->dst); bool replied = false; if (br_is_neigh_suppress_enabled(dst, vid)) { if (vid != 0) - br_nd_send(br, p, skb, n, + br_nd_send(br, p, skb, n, ha, skb->vlan_proto, skb_vlan_tag_get(skb)); else - br_nd_send(br, p, skb, n, 0, 0); + br_nd_send(br, p, skb, n, ha, 0, 0); replied = true; } diff --git a/net/bridge/br_netlink_tunnel.c b/net/bridge/br_netlink_tunnel.c index a713668ea34f0e..e7eceab5b515d4 100644 --- a/net/bridge/br_netlink_tunnel.c +++ b/net/bridge/br_netlink_tunnel.c @@ -302,7 +302,8 @@ int br_process_vlan_tunnel_info(const struct net_bridge *br, if (!(tinfo_last->flags & BRIDGE_VLAN_INFO_RANGE_BEGIN)) return -EINVAL; - if ((tinfo_curr->vid - tinfo_last->vid) != + if (tinfo_curr->vid < tinfo_last->vid || + (tinfo_curr->vid - tinfo_last->vid) != (tinfo_curr->tunid - tinfo_last->tunid)) return -EINVAL; t = tinfo_last->tunid; diff --git a/net/bridge/br_vlan.c b/net/bridge/br_vlan.c index 31c1b2cf75d923..1e0e436629ece5 100644 --- a/net/bridge/br_vlan.c +++ b/net/bridge/br_vlan.c @@ -1136,7 +1136,7 @@ int __br_vlan_set_default_pvid(struct net_bridge *br, u16 pvid, if (err) goto out; - if (br_vlan_delete(br, old_pvid)) + if (!br_vlan_delete(br, old_pvid)) br_vlan_notify(br, NULL, old_pvid, 0, RTM_DELVLAN); br_vlan_notify(br, NULL, pvid, 0, RTM_NEWVLAN); __set_bit(0, changed); @@ -1158,7 +1158,7 @@ int __br_vlan_set_default_pvid(struct net_bridge *br, u16 pvid, &vlchange, extack); if (err) goto err_port; - if (nbp_vlan_delete(p, old_pvid)) + if (!nbp_vlan_delete(p, old_pvid)) br_vlan_notify(br, p, old_pvid, 0, RTM_DELVLAN); br_vlan_notify(p->br, p, pvid, 0, RTM_NEWVLAN); __set_bit(p->port_no, changed); diff --git a/net/ceph/messenger_v2.c b/net/ceph/messenger_v2.c index 05f6eea299fc01..b323b61e702397 100644 --- a/net/ceph/messenger_v2.c +++ b/net/ceph/messenger_v2.c @@ -2142,6 +2142,11 @@ static int process_banner_prefix(struct ceph_connection *con) payload_len = ceph_decode_16(&p); dout("%s con %p payload_len %d\n", __func__, con, payload_len); + if (payload_len < sizeof(u64) + sizeof(u64)) { + con->error_msg = "protocol error, bad banner payload len"; + return -EINVAL; + } + return prepare_read_banner_payload(con, payload_len); } diff --git a/net/core/dev.c b/net/core/dev.c index ece6700536d9ba..39807b68ff2609 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -10333,6 +10333,37 @@ static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode, netdev_assert_locked_ops_compat(dev); + if (prog) { + enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB + ? XDP_MODE_DRV : XDP_MODE_SKB; + bool offload = mode == XDP_MODE_HW; + + if (!offload && dev_xdp_prog(dev, other_mode)) { + NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time"); + return -EEXIST; + } + if (!offload && bpf_prog_is_offloaded(prog->aux)) { + NL_SET_ERR_MSG(extack, "Using offloaded program without HW_MODE flag is not supported"); + return -EINVAL; + } + if (bpf_prog_is_dev_bound(prog->aux) && !bpf_offload_dev_match(prog, dev)) { + NL_SET_ERR_MSG(extack, "Program bound to different device"); + return -EINVAL; + } + if (bpf_prog_is_dev_bound(prog->aux) && mode == XDP_MODE_SKB) { + NL_SET_ERR_MSG(extack, "Can't attach device-bound programs in generic mode"); + return -EINVAL; + } + if (prog->expected_attach_type == BPF_XDP_DEVMAP) { + NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device"); + return -EINVAL; + } + if (prog->expected_attach_type == BPF_XDP_CPUMAP) { + NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device"); + return -EINVAL; + } + } + if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && prog && !prog->aux->xdp_has_frags) { NL_SET_ERR_MSG(extack, "unable to install XDP to device using tcp-data-split"); @@ -10472,38 +10503,10 @@ static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack new_prog = link->link.prog; if (new_prog) { - bool offload = mode == XDP_MODE_HW; - enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB - ? XDP_MODE_DRV : XDP_MODE_SKB; - if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) { NL_SET_ERR_MSG(extack, "XDP program already attached"); return -EBUSY; } - if (!offload && dev_xdp_prog(dev, other_mode)) { - NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time"); - return -EEXIST; - } - if (!offload && bpf_prog_is_offloaded(new_prog->aux)) { - NL_SET_ERR_MSG(extack, "Using offloaded program without HW_MODE flag is not supported"); - return -EINVAL; - } - if (bpf_prog_is_dev_bound(new_prog->aux) && !bpf_offload_dev_match(new_prog, dev)) { - NL_SET_ERR_MSG(extack, "Program bound to different device"); - return -EINVAL; - } - if (bpf_prog_is_dev_bound(new_prog->aux) && mode == XDP_MODE_SKB) { - NL_SET_ERR_MSG(extack, "Can't attach device-bound programs in generic mode"); - return -EINVAL; - } - if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) { - NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device"); - return -EINVAL; - } - if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) { - NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device"); - return -EINVAL; - } } /* don't call drivers if the effective program didn't change */ @@ -12164,6 +12167,7 @@ free_pcpu: free_percpu(dev->pcpu_refcnt); free_dev: #endif + ref_tracker_dir_exit(&dev->refcnt_tracker); kvfree(dev); return NULL; } diff --git a/net/core/filter.c b/net/core/filter.c index 16845987b24491..1e80a52ef86d08 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -8350,10 +8350,8 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_CGROUP_UNIX_SENDMSG: case BPF_CGROUP_INET4_GETPEERNAME: case BPF_CGROUP_INET6_GETPEERNAME: - case BPF_CGROUP_UNIX_GETPEERNAME: case BPF_CGROUP_INET4_GETSOCKNAME: case BPF_CGROUP_INET6_GETSOCKNAME: - case BPF_CGROUP_UNIX_GETSOCKNAME: return &bpf_sock_addr_setsockopt_proto; default: return NULL; @@ -8373,10 +8371,8 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_CGROUP_UNIX_SENDMSG: case BPF_CGROUP_INET4_GETPEERNAME: case BPF_CGROUP_INET6_GETPEERNAME: - case BPF_CGROUP_UNIX_GETPEERNAME: case BPF_CGROUP_INET4_GETSOCKNAME: case BPF_CGROUP_INET6_GETSOCKNAME: - case BPF_CGROUP_UNIX_GETSOCKNAME: return &bpf_sock_addr_getsockopt_proto; default: return NULL; diff --git a/net/core/lwt_bpf.c b/net/core/lwt_bpf.c index 652952d416f238..da49364ec63de7 100644 --- a/net/core/lwt_bpf.c +++ b/net/core/lwt_bpf.c @@ -167,10 +167,10 @@ static int bpf_output(struct net *net, struct sock *sk, struct sk_buff *skb) return dst->lwtstate->orig_output(net, sk, skb); } -static int xmit_check_hhlen(struct sk_buff *skb, int hh_len) +static int xmit_check_headroom(struct sk_buff *skb, int hroom) { - if (skb_headroom(skb) < hh_len) { - int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); + if (skb_headroom(skb) < hroom) { + int nhead = hroom - skb_headroom(skb); if (pskb_expand_head(skb, nhead, 0, GFP_ATOMIC)) return -ENOMEM; @@ -282,7 +282,7 @@ static int bpf_xmit(struct sk_buff *skb) bpf = bpf_lwt_lwtunnel(dst->lwtstate); if (bpf->xmit.prog) { - int hh_len = dst->dev->hard_header_len; + int hroom = LL_RESERVED_SPACE(dst->dev); __be16 proto = skb->protocol; int ret; @@ -298,9 +298,12 @@ static int bpf_xmit(struct sk_buff *skb) return -EINVAL; } /* If the header was expanded, headroom might be too - * small for L2 header to come, expand as needed. + * small for the L2 header to come, expand as needed. + * neigh_hh_output() copies the cached header in + * HH_DATA_MOD aligned chunks, so match the reservation + * made before LWT xmit. */ - ret = xmit_check_hhlen(skb, hh_len); + ret = xmit_check_headroom(skb, hroom); if (unlikely(ret)) return ret; diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c index d9dafe24f57e3f..af26bb77991b27 100644 --- a/net/core/net_namespace.c +++ b/net/core/net_namespace.c @@ -399,14 +399,8 @@ static __net_init void preinit_net_sysctl(struct net *net) } /* init code that must occur even if setup_net() is not called. */ -static __net_init int preinit_net(struct net *net, struct user_namespace *user_ns) +static __net_init void preinit_net(struct net *net, struct user_namespace *user_ns) { - int ret; - - ret = ns_common_init(net); - if (ret) - return ret; - refcount_set(&net->passive, 1); ref_tracker_dir_init(&net->refcnt_tracker, 128, "net_refcnt"); ref_tracker_dir_init(&net->notrefcnt_tracker, 128, "net_notrefcnt"); @@ -427,7 +421,6 @@ static __net_init int preinit_net(struct net *net, struct user_namespace *user_n INIT_LIST_HEAD(&net->ptype_all); INIT_LIST_HEAD(&net->ptype_specific); preinit_net_sysctl(net); - return 0; } /* @@ -532,8 +525,12 @@ void net_passive_dec(struct net *net) if (refcount_dec_and_test(&net->passive)) { kfree(rcu_access_pointer(net->gen)); +#ifdef CONFIG_REF_TRACKER /* There should not be any trackers left there. */ ref_tracker_dir_exit(&net->notrefcnt_tracker); + if (!net->refcnt_tracker.dead) + ref_tracker_dir_exit(&net->refcnt_tracker); +#endif /* Wait for an extra rcu_barrier() before final free. */ llist_add(&net->defer_free_list, &defer_free_list); @@ -566,12 +563,14 @@ struct net *copy_net_ns(u64 flags, goto dec_ucounts; } - rv = preinit_net(net, user_ns); - if (rv < 0) - goto dec_ucounts; + preinit_net(net, user_ns); net->ucounts = ucounts; get_user_ns(user_ns); + rv = ns_common_init(net); + if (rv) + goto put_userns_no_common; + rv = down_read_killable(&pernet_ops_rwsem); if (rv < 0) goto put_userns; @@ -583,6 +582,7 @@ struct net *copy_net_ns(u64 flags, if (rv < 0) { put_userns: ns_common_free(net); +put_userns_no_common: #ifdef CONFIG_KEYS key_remove_domain(net->key_domain); #endif @@ -1285,7 +1285,8 @@ void __init net_ns_init(void) * This currently cannot fail as the initial network namespace * has a static inode number. */ - if (preinit_net(&init_net, &init_user_ns)) + preinit_net(&init_net, &init_user_ns); + if (ns_common_init(&init_net)) panic("Could not preinitialize the initial network namespace"); down_write(&pernet_ops_rwsem); diff --git a/net/core/page_pool.c b/net/core/page_pool.c index 21dc4a9c8714df..8f8956fb061be1 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -484,6 +484,13 @@ static int page_pool_register_dma_index(struct page_pool *pool, if (unlikely(!PP_DMA_INDEX_BITS)) goto out; + /* + * Drivers request GFP flags according to both the current context and + * the device constraints, but the XArray entry itself is by no mean + * used by the device, so remove zone/policy flags. + */ + gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM | __GFP_COMP); + if (in_softirq()) err = xa_alloc(&pool->dma_mapped, &id, netmem_to_page(netmem), PP_DMA_INDEX_LIMIT, gfp); @@ -500,29 +507,40 @@ out: return err; } -static int page_pool_release_dma_index(struct page_pool *pool, - netmem_ref netmem) +static void __page_pool_unmap_netmem_dma(struct page_pool *pool, + netmem_ref netmem) { struct page *old, *page = netmem_to_page(netmem); unsigned long id; + dma_addr_t dma; - if (unlikely(!PP_DMA_INDEX_BITS)) - return 0; - - id = netmem_get_dma_index(netmem); - if (!id) - return -1; + if (!pool->dma_map) + return; - if (in_softirq()) - old = xa_cmpxchg(&pool->dma_mapped, id, page, NULL, 0); - else - old = xa_cmpxchg_bh(&pool->dma_mapped, id, page, NULL, 0); - if (old != page) - return -1; + /* Cache dma_addr before xa_cmpxchg. The scrub path holds no page ref; + * the unref path calls put_page() regardless of cmpxchg outcome, so + * after the cmpxchg we cannot safely touch netmem fields. + */ + dma = page_pool_get_dma_addr_netmem(netmem); - netmem_set_dma_index(netmem, 0); + if (likely(PP_DMA_INDEX_BITS)) { + id = netmem_get_dma_index(netmem); + if (!id) + return; + + if (in_softirq()) + old = xa_cmpxchg(&pool->dma_mapped, + id, page, NULL, 0); + else + old = xa_cmpxchg_bh(&pool->dma_mapped, + id, page, NULL, 0); + if (old != page) + return; + } - return 0; + dma_unmap_page_attrs(pool->p.dev, dma, + PAGE_SIZE << pool->p.order, pool->p.dma_dir, + DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); } static bool page_pool_dma_map(struct page_pool *pool, netmem_ref netmem, gfp_t gfp) @@ -728,24 +746,16 @@ void page_pool_clear_pp_info(netmem_ref netmem) static __always_inline void __page_pool_release_netmem_dma(struct page_pool *pool, netmem_ref netmem) { - dma_addr_t dma; - + /* Caller must hold a page ref: __page_pool_unmap_netmem_dma() is + * safe without a ref, but the field clears below require it. + */ if (!pool->dma_map) - /* Always account for inflight pages, even if we didn't - * map them - */ return; - if (page_pool_release_dma_index(pool, netmem)) - return; - - dma = page_pool_get_dma_addr_netmem(netmem); - - /* When page is unmapped, it cannot be returned to our pool */ - dma_unmap_page_attrs(pool->p.dev, dma, - PAGE_SIZE << pool->p.order, pool->p.dma_dir, - DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); + __page_pool_unmap_netmem_dma(pool, netmem); page_pool_set_dma_addr_netmem(netmem, 0); + if (likely(PP_DMA_INDEX_BITS)) + netmem_set_dma_index(netmem, 0); } /* Disconnects a page (from a page_pool). API users can have a need @@ -1171,8 +1181,9 @@ static void page_pool_scrub(struct page_pool *pool) synchronize_net(); } + /* No page ref, dma-unmap only. */ xa_for_each(&pool->dma_mapped, id, ptr) - __page_pool_release_netmem_dma(pool, page_to_netmem((struct page *)ptr)); + __page_pool_unmap_netmem_dma(pool, page_to_netmem((struct page *)ptr)); } /* No more consumers should exist, but producers could still diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 04776a1123342d..b3edb90a21d625 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -2004,12 +2004,12 @@ int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) int i, order, psize, new_frags; u32 d_off; - if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) - return -EINVAL; - if (!skb_frags_readable(skb)) return -EFAULT; + if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) + return -EINVAL; + if (!num_frags) goto release; diff --git a/net/hsr/hsr_device.c b/net/hsr/hsr_device.c index 5555b71ab19b59..9c3078dd38c2c0 100644 --- a/net/hsr/hsr_device.c +++ b/net/hsr/hsr_device.c @@ -820,6 +820,8 @@ err_unregister: hsr_del_ports(hsr); err_add_master: hsr_del_self_node(hsr); + hsr_del_nodes(&hsr->node_db); + hsr_del_nodes(&hsr->proxy_node_db); if (unregister) unregister_netdevice(hsr_dev); diff --git a/net/ife/ife.c b/net/ife/ife.c index 7a75947a31e310..2ddf725d3389ec 100644 --- a/net/ife/ife.c +++ b/net/ife/ife.c @@ -37,7 +37,7 @@ void *ife_encode(struct sk_buff *skb, u16 metalen) * where ORIGDATA = original ethernet header ... */ int hdrm = metalen + IFE_METAHDRLEN; - int total_push = hdrm + skb->dev->hard_header_len; + int total_push = hdrm + ETH_HLEN; struct ifeheadr *ifehdr; struct ethhdr *iethh; /* inner ether header */ int skboff = 0; @@ -50,9 +50,9 @@ void *ife_encode(struct sk_buff *skb, u16 metalen) iethh = (struct ethhdr *) skb->data; __skb_push(skb, total_push); - memcpy(skb->data, iethh, skb->dev->hard_header_len); + memcpy(skb->data, iethh, ETH_HLEN); skb_reset_mac_header(skb); - skboff += skb->dev->hard_header_len; + skboff += ETH_HLEN; /* total metadata length */ ifehdr = (struct ifeheadr *) (skb->data + skboff); @@ -69,12 +69,12 @@ void *ife_decode(struct sk_buff *skb, u16 *metalen) int total_pull; u16 ifehdrln; - if (!pskb_may_pull(skb, skb->dev->hard_header_len + IFE_METAHDRLEN)) + if (!pskb_may_pull(skb, ETH_HLEN + IFE_METAHDRLEN)) return NULL; - ifehdr = (struct ifeheadr *) (skb->data + skb->dev->hard_header_len); + ifehdr = (struct ifeheadr *)(skb->data + ETH_HLEN); ifehdrln = ntohs(ifehdr->metalen); - total_pull = skb->dev->hard_header_len + ifehdrln; + total_pull = ETH_HLEN + ifehdrln; if (unlikely(ifehdrln < 2)) return NULL; @@ -82,7 +82,7 @@ void *ife_decode(struct sk_buff *skb, u16 *metalen) if (unlikely(!pskb_may_pull(skb, total_pull + ETH_HLEN))) return NULL; - ifehdr = (struct ifeheadr *)(skb->data + skb->dev->hard_header_len); + ifehdr = (struct ifeheadr *)(skb->data + ETH_HLEN); skb_set_mac_header(skb, total_pull); __skb_pull(skb, total_pull); *metalen = ifehdrln - IFE_METAHDRLEN; diff --git a/net/ipv4/inetpeer.c b/net/ipv4/inetpeer.c index 5b957a831e7c39..adf6dc8a95b4f3 100644 --- a/net/ipv4/inetpeer.c +++ b/net/ipv4/inetpeer.c @@ -21,6 +21,7 @@ #include #include #include +#include /* * Theory of operations. @@ -52,6 +53,34 @@ */ static struct kmem_cache *peer_cachep __ro_after_init; +static siphash_aligned_key_t inetpeer_hash_key __read_mostly; + +static u64 inetpeer_addr_hash(const struct inetpeer_addr *a) +{ + net_get_random_once(&inetpeer_hash_key, sizeof(inetpeer_hash_key)); + + if (a->family == AF_INET) + return siphash_2u32((__force u32)a->a4.addr, a->a4.vif, + &inetpeer_hash_key); + + return siphash_4u32((__force u32)a->a6.s6_addr32[0], + (__force u32)a->a6.s6_addr32[1], + (__force u32)a->a6.s6_addr32[2], + (__force u32)a->a6.s6_addr32[3], + &inetpeer_hash_key); +} + +static int inetpeer_entry_cmp(u64 dhash, + const struct inetpeer_addr *daddr, + const struct inet_peer *p) +{ + if (dhash < p->hash) + return -1; + if (dhash > p->hash) + return 1; + + return inetpeer_addr_cmp(daddr, &p->daddr); +} void inet_peer_base_init(struct inet_peer_base *bp) { @@ -84,6 +113,7 @@ void __init inet_initpeers(void) /* Called with rcu_read_lock() or base->lock held */ static struct inet_peer *lookup(const struct inetpeer_addr *daddr, + u64 dhash, struct inet_peer_base *base, unsigned int seq, struct inet_peer *gc_stack[], @@ -105,7 +135,7 @@ static struct inet_peer *lookup(const struct inetpeer_addr *daddr, break; parent = next; p = rb_entry(parent, struct inet_peer, rb_node); - cmp = inetpeer_addr_cmp(daddr, &p->daddr); + cmp = inetpeer_entry_cmp(dhash, daddr, p); if (cmp == 0) { now = jiffies; if (READ_ONCE(p->dtime) != now) @@ -170,6 +200,7 @@ struct inet_peer *inet_getpeer(struct inet_peer_base *base, const struct inetpeer_addr *daddr) { struct inet_peer *p, *gc_stack[PEER_MAX_GC]; + u64 dhash = inetpeer_addr_hash(daddr); struct rb_node **pp, *parent; unsigned int gc_cnt, seq; @@ -177,7 +208,7 @@ struct inet_peer *inet_getpeer(struct inet_peer_base *base, * Because of a concurrent writer, we might not find an existing entry. */ seq = read_seqbegin(&base->lock); - p = lookup(daddr, base, seq, NULL, &gc_cnt, &parent, &pp); + p = lookup(daddr, dhash, base, seq, NULL, &gc_cnt, &parent, &pp); /* Make sure tree was not modified during our lookup. */ if (p && !read_seqretry(&base->lock, seq)) @@ -190,11 +221,12 @@ struct inet_peer *inet_getpeer(struct inet_peer_base *base, write_seqlock_bh(&base->lock); gc_cnt = 0; - p = lookup(daddr, base, seq, gc_stack, &gc_cnt, &parent, &pp); + p = lookup(daddr, dhash, base, seq, gc_stack, &gc_cnt, &parent, &pp); if (!p) { p = kmem_cache_alloc(peer_cachep, GFP_ATOMIC); if (p) { p->daddr = *daddr; + p->hash = dhash; p->dtime = (__u32)jiffies; refcount_set(&p->refcnt, 1); atomic_set(&p->rid, 0); diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 455441f1b69490..10f58281be17be 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1169,8 +1169,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) zc = MSG_SPLICE_PAGES; } - if (!sockc_err && sockc.dmabuf_id && - (!(flags & MSG_ZEROCOPY) || !sock_flag(sk, SOCK_ZEROCOPY))) { + if (!sockc_err && sockc.dmabuf_id && (zc != MSG_ZEROCOPY || !binding)) { err = -EINVAL; goto out_err; } @@ -1240,7 +1239,8 @@ restart: trace_tcp_sendmsg_locked(sk, msg, skb, size_goal); - if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) { + if (copy <= 0 || !tcp_skb_can_collapse_to(skb) || + unlikely(skb_frags_readable(skb) != !binding)) { bool first_skb; new_segment: diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 7b761edf86ee91..8d243e91761a92 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -3603,7 +3603,7 @@ start: avail_wnd = cur_mss; } - len = cur_mss * segs; + len = cur_mss * (tcp_urg_mode(tp) ? 1 : segs); if (len > avail_wnd) { len = rounddown(avail_wnd, cur_mss); if (!len) diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c index 604a58838901a7..3f2ed9b77deb51 100644 --- a/net/ipv6/ip6mr.c +++ b/net/ipv6/ip6mr.c @@ -1162,10 +1162,10 @@ static int ip6mr_cache_report(const struct mr_table *mrt, struct sk_buff *pkt, msg->im6_src = ipv6_hdr(pkt)->saddr; msg->im6_dst = ipv6_hdr(pkt)->daddr; - skb_dst_set(skb, dst_clone(skb_dst(pkt))); skb->ip_summed = CHECKSUM_UNNECESSARY; } + skb_dst_drop(skb); mrt6msg_netlink_event(mrt, skb); /* Deliver to user space multicast routing algorithms */ @@ -1246,6 +1246,7 @@ static int ip6mr_cache_unresolved(struct mr_table *mrt, mifi_t mifi, skb->skb_iif = dev->ifindex; } + skb_dst_drop(skb); skb_queue_tail(&c->_c.mfc_un.unres.unresolved, skb); spin_unlock_bh(&mfc_unres_lock); diff --git a/net/ipv6/route.c b/net/ipv6/route.c index a698f322db1cac..dc9060af5bb1e8 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -4840,7 +4840,7 @@ static void rt6_upper_bound_set(struct fib6_info *rt, int *weight, int total) { int upper_bound = -1; - if (!rt6_is_dead(rt)) { + if (total && !rt6_is_dead(rt)) { *weight += rt->fib6_nh->fib_nh_weight; upper_bound = DIV_ROUND_CLOSEST_ULL((u64) (*weight) << 31, total) - 1; diff --git a/net/ipv6/xfrm6_input.c b/net/ipv6/xfrm6_input.c index 89d0443b530732..07edef25898441 100644 --- a/net/ipv6/xfrm6_input.c +++ b/net/ipv6/xfrm6_input.c @@ -247,7 +247,7 @@ int xfrm6_input_addr(struct sk_buff *skb, xfrm_address_t *daddr, goto drop; } - if (1 + sp->len == XFRM_MAX_DEPTH) { + if (sp->len >= XFRM_MAX_DEPTH) { XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); goto drop; } diff --git a/net/kcm/kcmsock.c b/net/kcm/kcmsock.c index d469abcd989b4e..71af69d442f211 100644 --- a/net/kcm/kcmsock.c +++ b/net/kcm/kcmsock.c @@ -5,6 +5,7 @@ * Copyright (c) 2016 Tom Herbert */ +#include #include #include #include @@ -391,7 +392,9 @@ static int kcm_parse_func_strparser(struct strparser *strp, struct sk_buff *skb) struct bpf_prog *prog = psock->bpf_prog; int res; + rcu_read_lock(); res = bpf_prog_run_pin_on_cpu(prog, skb); + rcu_read_unlock(); return res; } diff --git a/net/mac80211/link.c b/net/mac80211/link.c index dc68144dc363af..931950a1050880 100644 --- a/net/mac80211/link.c +++ b/net/mac80211/link.c @@ -360,7 +360,8 @@ static int ieee80211_vif_update_links(struct ieee80211_sub_if_data *sdata, link = links[link_id]; ieee80211_link_init(sdata, link_id, &link->data, &link->conf); ieee80211_link_setup(&link->data); - ieee80211_set_wmm_default(&link->data, true, non_sta); + if (sdata->vif.type != NL80211_IFTYPE_AP_VLAN) + ieee80211_set_wmm_default(&link->data, true, non_sta); } if (new_links == 0) diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index fa773f3b0541ac..1b9d272e7f8e16 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -7977,7 +7977,7 @@ ieee80211_mgd_check_cross_link_csa(struct ieee80211_sub_if_data *sdata, prof = (void *)sta_profiles[link_id]; prof_elems = ieee802_11_parse_elems(prof->variable + (prof->sta_info_len - 1), - len - + len - sizeof(*prof) - (prof->sta_info_len - 1), IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_BEACON, diff --git a/net/mac80211/s1g.c b/net/mac80211/s1g.c index abc338e22e59cc..bd7baf48183677 100644 --- a/net/mac80211/s1g.c +++ b/net/mac80211/s1g.c @@ -147,7 +147,7 @@ ieee80211_s1g_tx_twt_setup_fail(struct ieee80211_sub_if_data *sdata, drv_twt_teardown_request(sdata->local, sdata, &sta->sta, flowid); - ieee80211_s1g_send_twt_teardown(sdata, mgmt->sa, sdata->vif.addr, + ieee80211_s1g_send_twt_teardown(sdata, mgmt->da, sdata->vif.addr, flowid); } diff --git a/net/mac80211/spectmgmt.c b/net/mac80211/spectmgmt.c index ec622750e1c9d5..880f4625775dce 100644 --- a/net/mac80211/spectmgmt.c +++ b/net/mac80211/spectmgmt.c @@ -227,7 +227,7 @@ int ieee80211_parse_ch_switch_ie(struct ieee80211_sub_if_data *sdata, { enum nl80211_band new_band = current_band; int new_freq; - u8 new_chan_no = 0, new_op_class = 0; + int new_chan_no = -1, new_op_class = -1; struct ieee80211_channel *new_chan; struct cfg80211_chan_def new_chandef = {}; const struct ieee80211_sec_chan_offs_ie *sec_chan_offs; @@ -256,7 +256,7 @@ int ieee80211_parse_ch_switch_ie(struct ieee80211_sub_if_data *sdata, new_op_class = ext_chansw_elem->new_operating_class; if (!ieee80211_operating_class_to_band(new_op_class, &new_band)) { - new_op_class = 0; + new_op_class = -1; if (!unprot_action) sdata_info(sdata, "cannot understand ECSA IE operating class, %d, ignoring\n", @@ -268,14 +268,14 @@ int ieee80211_parse_ch_switch_ie(struct ieee80211_sub_if_data *sdata, } } - if (!new_op_class && elems->ch_switch_ie) { + if (new_op_class < 0 && elems->ch_switch_ie) { new_chan_no = elems->ch_switch_ie->new_ch_num; csa_ie->count = elems->ch_switch_ie->count; csa_ie->mode = elems->ch_switch_ie->mode; } /* nothing here we understand */ - if (!new_chan_no) + if (new_chan_no < 0) return 1; /* Mesh Channel Switch Parameters Element */ @@ -349,7 +349,8 @@ int ieee80211_parse_ch_switch_ie(struct ieee80211_sub_if_data *sdata, get_unaligned_le16(bwi->info.optional); } else if (!wide_bw_chansw_ie || !wbcs_elem_to_chandef(wide_bw_chansw_ie, &new_chandef)) { - if (!ieee80211_operating_class_to_chandef(new_op_class, new_chan, + if (new_op_class < 0 || + !ieee80211_operating_class_to_chandef(new_op_class, new_chan, &new_chandef)) new_chandef = csa_ie->chanreq.oper; } diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index 91b14112e24f08..fd4c379b3f2016 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -5249,7 +5249,8 @@ static void ieee80211_set_beacon_cntdwn(struct ieee80211_sub_if_data *sdata, if (sdata->vif.type == NL80211_IFTYPE_AP && resp) { u16 *resp_offsets = resp->cntdwn_counter_offsets; - resp->data[resp_offsets[i]] = count; + if (resp_offsets[i]) + resp->data[resp_offsets[i]] = count; } } } diff --git a/net/netfilter/ipset/ip_set_hash_gen.h b/net/netfilter/ipset/ip_set_hash_gen.h index f00c82acd7f053..80ca523f304bb0 100644 --- a/net/netfilter/ipset/ip_set_hash_gen.h +++ b/net/netfilter/ipset/ip_set_hash_gen.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -98,14 +99,34 @@ struct htable { #define IPSET_NET_COUNT 1 #endif -/* Book-keeping of the prefixes added to the set */ +/** + * struct net_prefix - Representation of a network prefix. + * @cidr: The CIDR prefix length. + * @count: Number of occurrences. + */ struct net_prefix { - u8 cidr; /* the cidr value */ - u32 count; /* number of elements of this cidr */ + u32 cidr:8; + u32 count:24; }; +#define CIDR_MAX_COUNT ((1 << 24) - 1) + +/** + * struct net_prefixes - A collection of network prefixes. + * @rcu: RCU head + * @seq: Sequence counter guarding in-place reordering of @nets + * @len: Number of entries in the array. + * @nets: Array of net_prefix structures (sorted by CIDR descending). + * + * @nets entries are updated in place under @set's lock. A single entry's + * cidr/count pair is always updated atomically via READ_ONCE()/WRITE_ONCE(), + * but removing an entry also shifts every following entry down by one slot. + * Lockless readers that scan the whole array (i.e. more than a single + * indexed slot) must use @seq to detect and retry across such a shift. + */ struct net_prefixes { struct rcu_head rcu; + seqcount_spinlock_t seq; u8 len; struct net_prefix nets[] __counted_by(len); }; @@ -143,8 +164,11 @@ htable_size(u8 hbits) #endif #define INIT_CIDR(n, host_mask) ({ \ - const struct net_prefixes *__n = rcu_dereference(n); \ - DCIDR_PUT((__n)->len ? (__n)->nets[0].cidr : host_mask);\ + const struct net_prefixes *__n = rcu_dereference(n); \ + struct net_prefix __p = \ + __n->len ? READ_ONCE(__n->nets[0]) \ + : (struct net_prefix){}; \ + DCIDR_PUT(__p.count ? __p.cidr : host_mask); \ }) #endif /* IP_SET_HASH_WITH_NETS */ @@ -318,27 +342,43 @@ struct mtype_resize_ad { }; #ifdef IP_SET_HASH_WITH_NETS -/* Network cidr size book keeping when the hash stores different - * sized networks. cidr == real cidr + 1 to support /0. +/** + * mtype_add_cidr - Add a CIDR entry to hash table bookkeeping + * @set: Pointer to the ip_set + * @h: Pointer to the htype + * @cidr: The CIDR prefix length + * @n: The index of the net_prefix array to add @cidr to + * + * Performs an update if @cidr is found, otherwise performs COW-style + * allocation and replacement via RCU. + * + * Return: 0 on success, negative error code on failure. */ static int mtype_add_cidr(struct ip_set *set, struct htype *h, u8 cidr, u8 n) { - struct net_prefixes *nets, *tmp; int i, j, found, len = 0, ret = 0; + struct net_prefixes *nets, *tmp; + struct net_prefix np; spin_lock_bh(&set->lock); nets = __ipset_dereference(h->rnets[n]); /* Add in increasing prefix order, so larger cidr first */ for (i = 0, found = -1; i < nets->len; i++) { - if (nets->nets[i].count) + np = READ_ONCE(nets->nets[i]); + if (np.count) len++; if (found != -1) { continue; - } else if (nets->nets[i].cidr < cidr) { + } else if (np.cidr < cidr) { found = i; - } else if (nets->nets[i].cidr == cidr) { - nets->nets[i].count++; + } else if (np.cidr == cidr) { + if (np.count < CIDR_MAX_COUNT) { + np.count++; + WRITE_ONCE(nets->nets[i], np); + } else { + ret = -EOVERFLOW; + } goto unlock; } } @@ -350,6 +390,7 @@ mtype_add_cidr(struct ip_set *set, struct htype *h, u8 cidr, u8 n) } tmp->len = len; + seqcount_spinlock_init(&tmp->seq, &set->lock); for (i = 0, j = 0; i < nets->len; i++) { if (i == found) { tmp->nets[j].cidr = cidr; @@ -371,42 +412,60 @@ unlock: return ret; } +/** + * mtype_del_cidr - Remove CIDR entry and maintain array integrity. + * @set: Pointer to the ip_set. + * @h: Pointer to the htype. + * @cidr: The CIDR prefix length. + * @n: The index of the net_prefix array to remove @cidr from + * + * If CIDR entry count falls to 0, this function performs a "shift-left" + * operation on all following elements. This ensures that the array remains + * contiguous and maintains its descending order by CIDR. The vacated slot + * at the end of the array is zeroed out (cidr=0, count=0). + */ static void mtype_del_cidr(struct ip_set *set, struct htype *h, u8 cidr, u8 n) { - struct net_prefixes *nets, *tmp; - u8 i, j, len = 0; + struct net_prefixes *nets; + struct net_prefix np; int found; + u8 i, j; + + BUILD_BUG_ON(sizeof(struct net_prefix) != sizeof(u32)); spin_lock_bh(&set->lock); nets = __ipset_dereference(h->rnets[n]); for (i = 0, found = -1; i < nets->len; i++) { - if (nets->nets[i].count) - len++; - if (nets->nets[i].cidr == cidr) + np = READ_ONCE(nets->nets[i]); + if (np.count && np.cidr == cidr) { + np.count--; found = i; + break; + } } if (unlikely(found == -1)) goto unlock; - nets->nets[found].count--; - if (nets->nets[found].count) - goto unlock; - len--; - tmp = kzalloc_flex(*tmp, nets, len, GFP_ATOMIC); - if (!tmp) - /* Leave a hole */ + if (np.count) { + WRITE_ONCE(nets->nets[found], np); goto unlock; + } - tmp->len = len; + write_seqcount_begin(&nets->seq); for (i = 0, j = 0; i < nets->len; i++) { - if (!nets->nets[i].count || i == found) + if (i == found) continue; - tmp->nets[j].cidr = nets->nets[i].cidr; - tmp->nets[j++].count = nets->nets[i].count; + + np = READ_ONCE(nets->nets[i]); + if (i != j) + WRITE_ONCE(nets->nets[j], np); + j++; } - rcu_assign_pointer(h->rnets[n], tmp); - kfree_rcu(nets, rcu); + + while (j < nets->len) + WRITE_ONCE(nets->nets[j++], (struct net_prefix){}); + write_seqcount_end(&nets->seq); unlock: spin_unlock_bh(&set->lock); } @@ -451,7 +510,7 @@ mtype_flush(struct ip_set *set) { struct htype *h = set->data; #ifdef IP_SET_HASH_WITH_NETS - struct net_prefixes *nets, *tmp; + struct net_prefixes *nets; #endif struct htable *t; struct hbucket *n; @@ -477,17 +536,15 @@ mtype_flush(struct ip_set *set) } #ifdef IP_SET_HASH_WITH_NETS for (i = 0; i < IPSET_NET_COUNT; i++) { - nets = ipset_dereference_nfnl(h->rnets[i]); - tmp = kzalloc_obj(*tmp, GFP_ATOMIC); - if (!tmp) { - u8 j; + u8 j; - for (j = 0; j < nets->len; j++) - nets->nets[j].count = 0; - } else { - rcu_assign_pointer(h->rnets[i], tmp); - kfree_rcu(nets, rcu); - } + spin_lock_bh(&set->lock); + nets = ipset_dereference_nfnl(h->rnets[i]); + write_seqcount_begin(&nets->seq); + for (j = 0; j < nets->len; j++) + WRITE_ONCE(nets->nets[j], (struct net_prefix){}); + write_seqcount_end(&nets->seq); + spin_unlock_bh(&set->lock); } #endif } @@ -1253,31 +1310,41 @@ mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d, #if IPSET_NET_COUNT == 2 struct net_prefixes *nets1; struct mtype_elem orig = *d; + unsigned int seq1; int ret, i, j, k; #else int ret, i, j; #endif - u32 key, multi = 0; + unsigned int seq0; + u32 key, multi; u8 pos; pr_debug("test by nets\n"); rcu_read_lock_bh(); +retry: + multi = 0; nets0 = rcu_dereference_bh(h->rnets[0]); + seq0 = read_seqcount_begin(&nets0->seq); #if IPSET_NET_COUNT == 2 nets1 = rcu_dereference_bh(h->rnets[1]); + seq1 = read_seqcount_begin(&nets1->seq); #endif for (j = 0; j < nets0->len && !multi; j++) { - if (!nets0->nets[j].count) + struct net_prefix p0 = READ_ONCE(nets0->nets[j]); + + if (!p0.count) continue; #if IPSET_NET_COUNT == 2 mtype_data_reset_elem(d, &orig); - mtype_data_netmask(d, nets0->nets[j].cidr, false); + mtype_data_netmask(d, p0.cidr, false); for (k = 0; k < nets1->len && !multi; k++) { - if (!nets1->nets[k].count) + struct net_prefix p1 = READ_ONCE(nets1->nets[k]); + + if (!p1.count) continue; - mtype_data_netmask(d, nets1->nets[k].cidr, true); + mtype_data_netmask(d, p1.cidr, true); #else - mtype_data_netmask(d, nets0->nets[j].cidr); + mtype_data_netmask(d, p0.cidr); #endif key = HKEY(d, h->initval, t->htable_bits); n = rcu_dereference_bh(hbucket(t, key)); @@ -1304,6 +1371,12 @@ mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d, } ret = 0; unlock: + if (read_seqcount_retry(&nets0->seq, seq0)) + goto retry; +#if IPSET_NET_COUNT == 2 + if (read_seqcount_retry(&nets1->seq, seq1)) + goto retry; +#endif rcu_read_unlock_bh(); return ret; } @@ -1707,6 +1780,7 @@ IPSET_TOKEN(HTYPE, _create)(struct net *net, struct ip_set *set, kfree(rcu_dereference_raw(h->rnets[--i])); goto free_hregion; } + seqcount_spinlock_init(&nets->seq, &set->lock); RCU_INIT_POINTER(h->rnets[i], nets); } #endif diff --git a/net/netfilter/ipset/ip_set_hash_netiface.c b/net/netfilter/ipset/ip_set_hash_netiface.c index b44b95f766b7ad..b602cc43565ddf 100644 --- a/net/netfilter/ipset/ip_set_hash_netiface.c +++ b/net/netfilter/ipset/ip_set_hash_netiface.c @@ -38,7 +38,6 @@ MODULE_ALIAS("ip_set_hash:net,iface"); #define HTYPE hash_netiface #define IP_SET_HASH_WITH_NETS #define IP_SET_HASH_WITH_MULTI -#define IP_SET_HASH_WITH_NET0 #define STRSCPY(a, b) strscpy(a, b, IFNAMSIZ) diff --git a/net/netfilter/ipset/ip_set_hash_netportnet.c b/net/netfilter/ipset/ip_set_hash_netportnet.c index 6291532be7a5a2..61af1ce2712798 100644 --- a/net/netfilter/ipset/ip_set_hash_netportnet.c +++ b/net/netfilter/ipset/ip_set_hash_netportnet.c @@ -36,7 +36,6 @@ MODULE_ALIAS("ip_set_hash:net,port,net"); #define IP_SET_HASH_WITH_PROTO #define IP_SET_HASH_WITH_NETS #define IPSET_NET_COUNT 2 -#define IP_SET_HASH_WITH_NET0 /* IPv4 variant */ diff --git a/net/netfilter/ipvs/ip_vs_ftp.c b/net/netfilter/ipvs/ip_vs_ftp.c index b315c608fda43e..9e3e005a826353 100644 --- a/net/netfilter/ipvs/ip_vs_ftp.c +++ b/net/netfilter/ipvs/ip_vs_ftp.c @@ -102,7 +102,7 @@ static int ip_vs_ftp_get_addrport(char *data, char *data_limit, char *s, c; unsigned char p[6]; char edelim; - __u16 hport; + __u32 hport; int i = 0; if (data_limit - data < plen) { @@ -144,7 +144,11 @@ static int ip_vs_ftp_get_addrport(char *data, char *data_limit, return -1; c = *data; if (isdigit(c)) { - p[i] = p[i]*10 + c - '0'; + unsigned int val = p[i] * 10 + c - '0'; + + if (val > 255) + return -1; + p[i] = val; } else if (c == ',' && i < 5) { i++; p[i] = 0; @@ -222,6 +226,8 @@ static int ip_vs_ftp_get_addrport(char *data, char *data_limit, if (!isdigit(*s)) break; hport = hport * 10 + *s - '0'; + if (hport > 65535) + return -1; } if (s == data_limit || !hport || *s != edelim) return -1; diff --git a/net/netfilter/nf_conntrack_helper.c b/net/netfilter/nf_conntrack_helper.c index 500509b17663ed..f776cc0009da2e 100644 --- a/net/netfilter/nf_conntrack_helper.c +++ b/net/netfilter/nf_conntrack_helper.c @@ -456,6 +456,15 @@ static bool expect_iter_me(struct nf_conntrack_expect *exp, void *data) return this == me; } +void nf_conntrack_helper_release(struct nf_conntrack_helper *me) +{ + nf_ct_expect_iterate_destroy(expect_iter_me, me); + + if (refcount_dec_and_test(&me->ct_refcnt)) + kfree_rcu(me, rcu); +} +EXPORT_SYMBOL_GPL(nf_conntrack_helper_release); + void nf_conntrack_helper_unregister(struct nf_conntrack_helper *me) { mutex_lock(&nf_ct_helper_mutex); @@ -471,10 +480,7 @@ void nf_conntrack_helper_unregister(struct nf_conntrack_helper *me) */ synchronize_rcu(); - nf_ct_expect_iterate_destroy(expect_iter_me, me); - - if (refcount_dec_and_test(&me->ct_refcnt)) - kfree_rcu(me, rcu); + nf_conntrack_helper_release(me); } EXPORT_SYMBOL_GPL(nf_conntrack_helper_unregister); diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c index 31cbb1b55b9e6c..11e873a0bd1490 100644 --- a/net/netfilter/nf_conntrack_netlink.c +++ b/net/netfilter/nf_conntrack_netlink.c @@ -3042,7 +3042,7 @@ ctnetlink_exp_dump_expect(struct sk_buff *skb, #endif if (nla_put_be32(skb, CTA_EXPECT_TIMEOUT, htonl(timeout)) || nla_put_be32(skb, CTA_EXPECT_ID, nf_expect_get_id(exp)) || - nla_put_be32(skb, CTA_EXPECT_FLAGS, htonl(exp->flags)) || + nla_put_be32(skb, CTA_EXPECT_FLAGS, htonl(exp->flags & NF_CT_EXPECT_MASK)) || nla_put_be32(skb, CTA_EXPECT_CLASS, htonl(exp->class))) goto nla_put_failure; diff --git a/net/netfilter/nf_nat_sip.c b/net/netfilter/nf_nat_sip.c index 133bd713fe0c29..8c412bcf6cff52 100644 --- a/net/netfilter/nf_nat_sip.c +++ b/net/netfilter/nf_nat_sip.c @@ -273,12 +273,17 @@ next: SIP_HDR_CONTACT, &in_header, &matchoff, &matchlen, &addr, &port) > 0) { + int old_len = skb->len, delta; + if (!map_addr(skb, protoff, dataoff, dptr, datalen, matchoff, matchlen, &addr, port)) { nf_ct_helper_log(skb, ct, "cannot mangle contact"); return NF_DROP; } + + delta = (int)skb->len - old_len; + coff += delta; } if (!map_sip_addr(skb, protoff, dataoff, dptr, datalen, SIP_HDR_FROM) || diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index 3a7c8f7a630415..63003a3b8babe6 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -6916,39 +6916,11 @@ static void nft_trans_elems_destroy(const struct nft_ctx *ctx, nf_tables_set_elem_destroy(ctx, te->set, te->elems[i].priv); } -int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set, - struct nft_expr *expr_array[]) -{ - struct nft_expr *expr; - int err, i, k; - - for (i = 0; i < set->num_exprs; i++) { - expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL_ACCOUNT); - if (!expr) - goto err_expr; - - err = nft_expr_clone(expr, set->exprs[i], GFP_KERNEL_ACCOUNT); - if (err < 0) { - kfree(expr); - goto err_expr; - } - expr_array[i] = expr; - } - - return 0; - -err_expr: - for (k = i - 1; k >= 0; k--) - nft_expr_destroy(ctx, expr_array[k]); - - return -ENOMEM; -} - static int nft_set_elem_expr_setup(struct nft_ctx *ctx, const struct nft_set_ext_tmpl *tmpl, const struct nft_set_ext *ext, struct nft_expr *expr_array[], - u32 num_exprs) + u32 num_exprs, bool override_exprs) { struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext); u32 len = sizeof(struct nft_set_elem_expr); @@ -6971,7 +6943,8 @@ static int nft_set_elem_expr_setup(struct nft_ctx *ctx, goto err_elem_expr_setup; elem_expr->size += expr_array[i]->ops->size; - nft_expr_destroy(ctx, expr_array[i]); + if (override_exprs) + nft_expr_destroy(ctx, expr_array[i]); expr_array[i] = NULL; } @@ -6979,7 +6952,9 @@ static int nft_set_elem_expr_setup(struct nft_ctx *ctx, err_elem_expr_setup: for (; i < num_exprs; i++) { - nft_expr_destroy(ctx, expr_array[i]); + if (override_exprs) + nft_expr_destroy(ctx, expr_array[i]); + expr_array[i] = NULL; } @@ -7258,6 +7233,7 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, struct nft_set_binding *binding; struct nft_elem_priv *elem_priv; struct nft_object *obj = NULL; + bool override_exprs = false; struct nft_userdata *udata; struct nft_data_desc desc; enum nft_registers dreg; @@ -7363,6 +7339,7 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, expr_array[0] = expr; num_exprs = 1; + override_exprs = true; if (set->num_exprs && set->exprs[0]->ops != expr->ops) { err = -EOPNOTSUPP; @@ -7391,6 +7368,7 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, } expr_array[i] = expr; num_exprs++; + override_exprs = true; if (set->num_exprs && expr->ops != set->exprs[i]->ops) { err = -EOPNOTSUPP; @@ -7404,9 +7382,8 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, } } else if (set->num_exprs > 0 && !(flags & NFT_SET_ELEM_INTERVAL_END)) { - err = nft_set_elem_expr_clone(ctx, set, expr_array); - if (err < 0) - goto err_set_elem_expr_clone; + for (i = 0; i < set->num_exprs; i++) + expr_array[i] = set->exprs[i]; num_exprs = set->num_exprs; } @@ -7545,7 +7522,8 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, udata->len = ulen - 1; nla_memcpy(&udata->data, nla[NFTA_SET_ELEM_USERDATA], ulen); } - err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs); + err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs, + override_exprs); if (err < 0) goto err_elem_free; @@ -7653,9 +7631,11 @@ err_parse_key_end: err_parse_key: nft_data_release(&elem.key.val, NFT_DATA_VALUE); err_set_elem_expr: - for (i = 0; i < num_exprs && expr_array[i]; i++) - nft_expr_destroy(ctx, expr_array[i]); -err_set_elem_expr_clone: + if (override_exprs) { + for (i = 0; i < num_exprs && expr_array[i]; i++) + nft_expr_destroy(ctx, expr_array[i]); + } + return err; } @@ -10941,10 +10921,6 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) return -EAGAIN; } - err = nft_flow_rule_offload_commit(net); - if (err < 0) - return err; - /* 1. Allocate space for next generation rules_gen_X[] */ list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) { struct nft_table *table = trans->table; @@ -10969,6 +10945,16 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) } } + /* must be last, so audit and chain blob set up does not leave hardware + * in consistent state. + */ + err = nft_flow_rule_offload_commit(net); + if (err < 0) { + nf_tables_commit_chain_prepare_cancel(net); + nf_tables_commit_audit_free(&adl); + return err; + } + /* step 2. Make rules_gen_X visible to packet path */ list_for_each_entry(table, &nft_net->tables, list) { list_for_each_entry(chain, &table->chains, list) diff --git a/net/netfilter/nft_ct.c b/net/netfilter/nft_ct.c index 03a88c77e0f028..9dbf127df9c81e 100644 --- a/net/netfilter/nft_ct.c +++ b/net/netfilter/nft_ct.c @@ -1213,6 +1213,8 @@ struct nft_ct_expect_obj { u8 l4proto; u8 size; u32 timeout; + + struct nf_conntrack_helper *helper; }; static int nft_ct_expect_timeout_get(const struct nlattr *attr, u32 *val) @@ -1226,6 +1228,93 @@ static int nft_ct_expect_timeout_get(const struct nlattr *attr, u32 *val) return 0; } +#if IS_ENABLED(CONFIG_NF_NAT) +static void nft_ct_nat_follow_master(struct nf_conn *ct, struct nf_conntrack_expect *this) +{ + const struct nf_ct_helper_expectfn *expfn; + + expfn = nf_ct_helper_expectfn_find_by_name("nat-follow-master"); + if (expfn) + expfn->expectfn(ct, this); +} +#endif + +struct nft_ct_expect_data { + struct nft_ct_expect_obj obj; + enum ip_conntrack_dir dir; +}; + +static int ct_expect_help(struct sk_buff *skb, unsigned int protoff, + struct nf_conn *ct, enum ip_conntrack_info ctinfo) +{ + enum ip_conntrack_dir dir = CTINFO2DIR(ctinfo); + struct nft_ct_expect_data *expect_data; + struct nf_conntrack_expect *exp; + int ret = NF_ACCEPT; + u16 l3num; + + if (nf_ct_is_confirmed(ct)) + return NF_ACCEPT; + + expect_data = nfct_help_data(ct); + if (!expect_data) + return NF_ACCEPT; + + if (expect_data->dir != dir) + return NF_ACCEPT; + + exp = nf_ct_expect_alloc(ct); + if (!exp) + return NF_DROP; + + if (expect_data->obj.l3num == NFPROTO_INET) + l3num = nf_ct_l3num(ct); + else + l3num = expect_data->obj.l3num; + + nf_ct_expect_init(exp, NF_CT_EXPECT_CLASS_DEFAULT, l3num, + &ct->tuplehash[!dir].tuple.src.u3, + &ct->tuplehash[!dir].tuple.dst.u3, + expect_data->obj.l4proto, NULL, &expect_data->obj.dport); + exp->timeout += expect_data->obj.timeout; + +#if IS_ENABLED(CONFIG_NF_NAT) + if (ct->status & IPS_NAT_MASK) { + exp->saved_proto.tcp.port = expect_data->obj.dport; + exp->dir = !dir; + exp->expectfn = nft_ct_nat_follow_master; + } +#endif + if (nf_ct_expect_related(exp, 0) != 0) + ret = NF_ACCEPT; + + nf_ct_expect_put(exp); + + return ret; +} + +static int nft_ct_expect_helper_alloc(struct nft_ct_expect_obj *priv) +{ + struct nf_conntrack_helper *ct_expect_helper; + + ct_expect_helper = kzalloc_obj(struct nf_conntrack_helper, + GFP_KERNEL_ACCOUNT); + if (!ct_expect_helper) + return -ENOMEM; + + snprintf(ct_expect_helper->name, sizeof(ct_expect_helper->name), "%s", + "nft_ct_expect"); + ct_expect_helper->me = THIS_MODULE; + ct_expect_helper->expect_policy[NF_CT_EXPECT_CLASS_DEFAULT].max_expected = priv->size; + rcu_assign_pointer(ct_expect_helper->help, ct_expect_help); + refcount_set(&ct_expect_helper->ct_refcnt, 1); + + /* No need to register this helper, this is internal. */ + priv->helper = ct_expect_helper; + + return 0; +} + static int nft_ct_expect_obj_init(const struct nft_ctx *ctx, const struct nlattr * const tb[], struct nft_object *obj) @@ -1233,6 +1322,8 @@ static int nft_ct_expect_obj_init(const struct nft_ctx *ctx, struct nft_ct_expect_obj *priv = nft_obj_data(obj); int err; + NF_CT_HELPER_BUILD_BUG_ON(sizeof(struct nft_ct_expect_data)); + if (!tb[NFTA_CT_EXPECT_L4PROTO] || !tb[NFTA_CT_EXPECT_DPORT] || !tb[NFTA_CT_EXPECT_TIMEOUT] || @@ -1272,13 +1363,31 @@ static int nft_ct_expect_obj_init(const struct nft_ctx *ctx, priv->dport = nla_get_be16(tb[NFTA_CT_EXPECT_DPORT]); priv->size = nla_get_u8(tb[NFTA_CT_EXPECT_SIZE]); + if (!priv->size) + priv->size = NF_CT_EXPECT_MAX_CNT; + + err = nf_ct_netns_get(ctx->net, ctx->family); + if (err < 0) + return err; - return nf_ct_netns_get(ctx->net, ctx->family); + err = nft_ct_expect_helper_alloc(priv); + if (err < 0) { + nf_ct_netns_put(ctx->net, ctx->family); + return err; + } + + return err; } static void nft_ct_expect_obj_destroy(const struct nft_ctx *ctx, - struct nft_object *obj) + struct nft_object *obj) { + const struct nft_ct_expect_obj *priv = nft_obj_data(obj); + struct nf_conntrack_helper *me = priv->helper; + + /* This helper is going away, disable it. */ + rcu_assign_pointer(me->help, NULL); + nf_conntrack_helper_release(me); nf_ct_netns_put(ctx->net, ctx->family); } @@ -1302,11 +1411,9 @@ static void nft_ct_expect_obj_eval(struct nft_object *obj, const struct nft_pktinfo *pkt) { const struct nft_ct_expect_obj *priv = nft_obj_data(obj); - struct nf_conntrack_expect *exp; + struct nft_ct_expect_data *expect_data; enum ip_conntrack_info ctinfo; struct nf_conn_help *help; - enum ip_conntrack_dir dir; - u16 l3num = priv->l3num; struct nf_conn *ct; ct = nf_ct_get(pkt->skb, &ctinfo); @@ -1314,38 +1421,30 @@ static void nft_ct_expect_obj_eval(struct nft_object *obj, regs->verdict.code = NFT_BREAK; return; } - dir = CTINFO2DIR(ctinfo); help = nfct_help(ct); - if (!help) - help = nf_ct_helper_ext_add(ct, GFP_ATOMIC); - if (!help) { - regs->verdict.code = NF_DROP; - return; - } - - if (help->expecting[NF_CT_EXPECT_CLASS_DEFAULT] >= priv->size) { + if (help) { regs->verdict.code = NFT_BREAK; return; } - if (l3num == NFPROTO_INET) - l3num = nf_ct_l3num(ct); - exp = nf_ct_expect_alloc(ct); - if (exp == NULL) { + help = nf_ct_helper_ext_add(ct, GFP_ATOMIC); + if (!help) { regs->verdict.code = NF_DROP; return; } - nf_ct_expect_init(exp, NF_CT_EXPECT_CLASS_DEFAULT, l3num, - &ct->tuplehash[!dir].tuple.src.u3, - &ct->tuplehash[!dir].tuple.dst.u3, - priv->l4proto, NULL, &priv->dport); - exp->timeout += priv->timeout; - if (nf_ct_expect_related(exp, 0) != 0) - regs->verdict.code = NF_DROP; + expect_data = nfct_help_data(ct); + if (!expect_data) { + regs->verdict.code = NFT_BREAK; + return; + } + expect_data->obj = *priv; + expect_data->obj.helper = NULL; + expect_data->dir = CTINFO2DIR(ctinfo); - nf_ct_expect_put(exp); + if (help && refcount_inc_not_zero(&priv->helper->ct_refcnt)) + rcu_assign_pointer(help->helper, priv->helper); } static const struct nla_policy nft_ct_expect_policy[NFTA_CT_EXPECT_MAX + 1] = { @@ -1375,6 +1474,13 @@ static struct nft_object_type nft_ct_expect_obj_type __read_mostly = { .owner = THIS_MODULE, }; +#if IS_ENABLED(CONFIG_NF_NAT) +static struct nf_ct_helper_expectfn nft_ct_nat __read_mostly = { + .name = "nft_ct-follow-master", + .expectfn = nft_ct_nat_follow_master, +}; +#endif + static int __init nft_ct_module_init(void) { int err; @@ -1400,6 +1506,9 @@ static int __init nft_ct_module_init(void) err = nft_register_obj(&nft_ct_timeout_obj_type); if (err < 0) goto err4; +#endif +#if IS_ENABLED(CONFIG_NF_NAT) + nf_ct_helper_expectfn_register(&nft_ct_nat); #endif return 0; @@ -1425,6 +1534,13 @@ static void __exit nft_ct_module_exit(void) nft_unregister_obj(&nft_ct_helper_obj_type); nft_unregister_expr(&nft_notrack_type); nft_unregister_expr(&nft_ct_type); + +#if IS_ENABLED(CONFIG_NF_NAT) + nf_ct_helper_expectfn_unregister(&nft_ct_nat); + synchronize_rcu(); + nf_ct_helper_expectfn_destroy(&nft_ct_nat); + synchronize_rcu(); +#endif } module_init(nft_ct_module_init); diff --git a/net/netfilter/nft_dynset.c b/net/netfilter/nft_dynset.c index ee9d3e7b1ecf1a..fa4da694e92d8f 100644 --- a/net/netfilter/nft_dynset.c +++ b/net/netfilter/nft_dynset.c @@ -19,7 +19,8 @@ struct nft_dynset { u8 sreg_key; u8 sreg_data; bool invert; - bool expr; + bool expr:1, + override_exprs:1; u8 num_exprs; u64 timeout; struct nft_expr *expr_array[NFT_SET_EXPR_MAX]; @@ -257,6 +258,7 @@ static int nft_dynset_init(const struct nft_ctx *ctx, priv->num_exprs++; priv->expr_array[0] = dynset_expr; + priv->override_exprs = true; if (set->num_exprs > 1 || (set->num_exprs == 1 && @@ -289,6 +291,7 @@ static int nft_dynset_init(const struct nft_ctx *ctx, } priv->expr_array[i] = dynset_expr; priv->num_exprs++; + priv->override_exprs = true; if (set->num_exprs) { if (i >= set->num_exprs) { @@ -307,9 +310,8 @@ static int nft_dynset_init(const struct nft_ctx *ctx, goto err_expr_free; } } else if (set->num_exprs > 0) { - err = nft_set_elem_expr_clone(ctx, set, priv->expr_array); - if (err < 0) - return err; + for (i = 0; i < set->num_exprs; i++) + priv->expr_array[i] = set->exprs[i]; priv->num_exprs = set->num_exprs; } @@ -339,8 +341,10 @@ static int nft_dynset_init(const struct nft_ctx *ctx, return 0; err_expr_free: - for (i = 0; i < priv->num_exprs; i++) - nft_expr_destroy(ctx, priv->expr_array[i]); + if (priv->override_exprs) { + for (i = 0; i < priv->num_exprs; i++) + nft_expr_destroy(ctx, priv->expr_array[i]); + } return err; } @@ -367,9 +371,10 @@ static void nft_dynset_destroy(const struct nft_ctx *ctx, struct nft_dynset *priv = nft_expr_priv(expr); int i; - for (i = 0; i < priv->num_exprs; i++) - nft_expr_destroy(ctx, priv->expr_array[i]); - + if (priv->override_exprs) { + for (i = 0; i < priv->num_exprs; i++) + nft_expr_destroy(ctx, priv->expr_array[i]); + } nf_tables_destroy_set(ctx, priv->set); } @@ -393,7 +398,7 @@ static int nft_dynset_dump(struct sk_buff *skb, nf_jiffies64_to_msecs(priv->timeout), NFTA_DYNSET_PAD)) goto nla_put_failure; - if (priv->set->num_exprs == 0) { + if (priv->set->num_exprs == 0 || priv->override_exprs) { if (priv->num_exprs == 1) { if (nft_expr_dump(skb, NFTA_DYNSET_EXPR, priv->expr_array[0], reset)) diff --git a/net/netfilter/xt_cgroup.c b/net/netfilter/xt_cgroup.c index 43d2ae2be628dc..28e6cd51b2fb0d 100644 --- a/net/netfilter/xt_cgroup.c +++ b/net/netfilter/xt_cgroup.c @@ -23,7 +23,7 @@ MODULE_DESCRIPTION("Xtables: process control group matching"); MODULE_ALIAS("ipt_cgroup"); MODULE_ALIAS("ip6t_cgroup"); -#define NET_CLS_CLASSID_INVALID_MSG "xt_cgroup: classid invalid without net_cls cgroups\n" +#define NET_CLS_CLASSID_INVALID_MSG "classid invalid without net_cls cgroups\n" static int cgroup_mt_check_v0(const struct xt_mtchk_param *par) { @@ -33,7 +33,7 @@ static int cgroup_mt_check_v0(const struct xt_mtchk_param *par) return -EINVAL; if (!IS_ENABLED(CONFIG_CGROUP_NET_CLASSID)) { - pr_info(NET_CLS_CLASSID_INVALID_MSG); + pr_info_ratelimited(NET_CLS_CLASSID_INVALID_MSG); return -EINVAL; } @@ -49,7 +49,7 @@ static int cgroup_mt_check_v1(const struct xt_mtchk_param *par) return -EINVAL; if (!info->has_path && !info->has_classid) { - pr_info("xt_cgroup: no path or classid specified\n"); + pr_info_ratelimited("no path or classid specified\n"); return -EINVAL; } @@ -59,7 +59,7 @@ static int cgroup_mt_check_v1(const struct xt_mtchk_param *par) } if (info->has_classid && !IS_ENABLED(CONFIG_CGROUP_NET_CLASSID)) { - pr_info(NET_CLS_CLASSID_INVALID_MSG); + pr_info_ratelimited(NET_CLS_CLASSID_INVALID_MSG); return -EINVAL; } @@ -89,7 +89,7 @@ static int cgroup_mt_check_v2(const struct xt_mtchk_param *par) return -EINVAL; if (!info->has_path && !info->has_classid) { - pr_info("xt_cgroup: no path or classid specified\n"); + pr_info_ratelimited("no path or classid specified\n"); return -EINVAL; } @@ -99,7 +99,7 @@ static int cgroup_mt_check_v2(const struct xt_mtchk_param *par) } if (info->has_classid && !IS_ENABLED(CONFIG_CGROUP_NET_CLASSID)) { - pr_info(NET_CLS_CLASSID_INVALID_MSG); + pr_info_ratelimited(NET_CLS_CLASSID_INVALID_MSG); return -EINVAL; } diff --git a/net/netfilter/xt_hl.c b/net/netfilter/xt_hl.c index 4a12a757ecbf86..59e93d97b5072c 100644 --- a/net/netfilter/xt_hl.c +++ b/net/netfilter/xt_hl.c @@ -28,7 +28,7 @@ static int ttl_mt_check(const struct xt_mtchk_param *par) const struct ipt_ttl_info *info = par->matchinfo; if (info->mode > IPT_TTL_GT) { - pr_err("Unknown TTL match mode: %d\n", info->mode); + pr_info_ratelimited("Unknown TTL match mode: %d\n", info->mode); return -EINVAL; } @@ -59,7 +59,7 @@ static int hl_mt6_check(const struct xt_mtchk_param *par) const struct ip6t_hl_info *info = par->matchinfo; if (info->mode > IP6T_HL_GT) { - pr_err("Unknown Hop Limit match mode: %d\n", info->mode); + pr_info_ratelimited("Unknown Hop Limit match mode: %d\n", info->mode); return -EINVAL; } diff --git a/net/nfc/digital_core.c b/net/nfc/digital_core.c index 7cb1e6aaae90da..18236221d89882 100644 --- a/net/nfc/digital_core.c +++ b/net/nfc/digital_core.c @@ -127,7 +127,7 @@ static void digital_wq_cmd_complete(struct work_struct *work) mutex_unlock(&ddev->cmd_lock); - if (!IS_ERR(cmd->resp)) + if (!IS_ERR_OR_NULL(cmd->resp)) print_hex_dump_debug("DIGITAL RX: ", DUMP_PREFIX_NONE, 16, 1, cmd->resp->data, cmd->resp->len, false); diff --git a/net/nfc/llcp_core.c b/net/nfc/llcp_core.c index 6ae0d64c17b93f..cac1b5487064d0 100644 --- a/net/nfc/llcp_core.c +++ b/net/nfc/llcp_core.c @@ -1288,10 +1288,9 @@ static void nfc_llcp_recv_snl(struct nfc_llcp_local *local, { struct nfc_llcp_sock *llcp_sock; u8 dsap, ssap, type, length, tid, sap; - const u8 *tlv; - u16 tlv_len, offset; + const u8 *tlv, *tlv_end; const char *service_name; - size_t service_name_len; + int service_name_len; struct nfc_llcp_sdp_tlv *sdp; HLIST_HEAD(llc_sdres_list); size_t sdres_tlvs_len; @@ -1307,22 +1306,34 @@ static void nfc_llcp_recv_snl(struct nfc_llcp_local *local, return; } + /* + * Walk the SNL TLV list in the linear part of the skb only, + * bounded by skb_tail_pointer(). Each TLV needs a two-byte + * header (type, length) and its declared length must fit before + * the end; this also keeps the walk safe for very short frames. + */ tlv = &skb->data[LLCP_HEADER_SIZE]; - tlv_len = skb->len - LLCP_HEADER_SIZE; - offset = 0; + tlv_end = skb_tail_pointer(skb); sdres_tlvs_len = 0; - while (offset < tlv_len) { + while (tlv + 2 < tlv_end) { type = tlv[0]; length = tlv[1]; + if (tlv + 2 + length > tlv_end) + break; + switch (type) { case LLCP_TLV_SDREQ: + if (length < 1) + break; + tid = tlv[2]; service_name = (char *) &tlv[3]; service_name_len = length - 1; - pr_debug("Looking for %.16s\n", service_name); + pr_debug("Looking for %.*s\n", service_name_len, + service_name); if (service_name_len == strlen("urn:nfc:sn:sdp") && !strncmp(service_name, "urn:nfc:sn:sdp", @@ -1382,6 +1393,9 @@ add_snl: break; case LLCP_TLV_SDRES: + if (length != 2) + break; + mutex_lock(&local->sdreq_lock); pr_debug("LLCP_TLV_SDRES: searching tid %d\n", tlv[2]); @@ -1410,7 +1424,6 @@ add_snl: break; } - offset += length + 2; tlv += length + 2; } diff --git a/net/nfc/llcp_sock.c b/net/nfc/llcp_sock.c index feab29fc62f44b..5558d8a4d48b36 100644 --- a/net/nfc/llcp_sock.c +++ b/net/nfc/llcp_sock.c @@ -319,14 +319,22 @@ static int nfc_llcp_getsockopt(struct socket *sock, int level, int optname, if (get_user(len, optlen)) return -EFAULT; - local = llcp_sock->local; - if (!local) - return -ENODEV; + if (len < 0) + return -EINVAL; + + if (len < sizeof(u32)) + return -EINVAL; len = min_t(u32, len, sizeof(u32)); lock_sock(sk); + local = llcp_sock->local; + if (!local) { + release_sock(sk); + return -ENODEV; + } + switch (optname) { case NFC_LLCP_RW: rw = llcp_sock->rw > LLCP_MAX_RW ? local->rw : llcp_sock->rw; diff --git a/net/nfc/nci/data.c b/net/nfc/nci/data.c index 5f98c73db5afde..4253edea5c8dbd 100644 --- a/net/nfc/nci/data.c +++ b/net/nfc/nci/data.c @@ -46,11 +46,11 @@ void nci_data_exchange_complete(struct nci_dev *ndev, struct sk_buff *skb, timer_delete_sync(&ndev->data_timer); clear_bit(NCI_DATA_EXCHANGE_TO, &ndev->flags); - /* Mark the exchange as done before calling the callback. - * The callback (e.g. rawsock_data_exchange_complete) may - * want to immediately queue another data exchange. - */ - clear_bit(NCI_DATA_EXCHANGE, &ndev->flags); + /* Claim completion atomically -- both close and rx_work may race here */ + if (!test_and_clear_bit(NCI_DATA_EXCHANGE, &ndev->flags)) { + kfree_skb(skb); + return; + } if (cb) { /* forward skb to nfc core */ diff --git a/net/nfc/nci/rsp.c b/net/nfc/nci/rsp.c index 165aa4115166a7..b0ab4f5acbce88 100644 --- a/net/nfc/nci/rsp.c +++ b/net/nfc/nci/rsp.c @@ -50,11 +50,27 @@ static u8 nci_core_init_rsp_packet_v1(struct nci_dev *ndev, const struct nci_core_init_rsp_1 *rsp_1 = (void *)skb->data; const struct nci_core_init_rsp_2 *rsp_2; + /* Ensure that the status field can be accessed. */ + if (skb_headlen(skb) < 1) + return NCI_STATUS_SYNTAX_ERROR; + pr_debug("status 0x%x\n", rsp_1->status); if (rsp_1->status != NCI_STATUS_OK) return rsp_1->status; + /* Success response must contain the full fixed-size header */ + if (skb_headlen(skb) < sizeof(*rsp_1)) + return NCI_STATUS_SYNTAX_ERROR; + + /* Ensure the variable-length rf_interfaces array and trailing + * rsp_2 structure are fully contained within the skb. + */ + if (skb_headlen(skb) < sizeof(*rsp_1) + + rsp_1->num_supported_rf_interfaces + + sizeof(*rsp_2)) + return NCI_STATUS_SYNTAX_ERROR; + ndev->nfcc_features = __le32_to_cpu(rsp_1->nfcc_features); ndev->num_supported_rf_interfaces = rsp_1->num_supported_rf_interfaces; @@ -87,15 +103,25 @@ static u8 nci_core_init_rsp_packet_v2(struct nci_dev *ndev, const struct sk_buff *skb) { const struct nci_core_init_rsp_nci_ver2 *rsp = (void *)skb->data; - const u8 *supported_rf_interface = rsp->supported_rf_interfaces; + const u8 *supported_rf_interface; u8 rf_interface_idx = 0; u8 rf_extension_cnt = 0; + /* Ensure that the status field can be accessed. */ + if (skb_headlen(skb) < 1) + return NCI_STATUS_SYNTAX_ERROR; + pr_debug("status %x\n", rsp->status); if (rsp->status != NCI_STATUS_OK) return rsp->status; + /* Success response must contain the full fixed-size header */ + if (skb_headlen(skb) < sizeof(*rsp)) + return NCI_STATUS_SYNTAX_ERROR; + + supported_rf_interface = rsp->supported_rf_interfaces; + ndev->nfcc_features = __le32_to_cpu(rsp->nfcc_features); ndev->num_supported_rf_interfaces = rsp->num_supported_rf_interfaces; @@ -104,13 +130,22 @@ static u8 nci_core_init_rsp_packet_v2(struct nci_dev *ndev, NCI_MAX_SUPPORTED_RF_INTERFACES); while (rf_interface_idx < ndev->num_supported_rf_interfaces) { - ndev->supported_rf_interfaces[rf_interface_idx++] = *supported_rf_interface++; + /* Each entry: [rf_interface_type (1B)] [ext_count (1B)] [ext...] */ + if (supported_rf_interface + 2 > skb_tail_pointer(skb)) + break; + ndev->supported_rf_interfaces[rf_interface_idx] = *supported_rf_interface++; - /* skip rf extension parameters */ rf_extension_cnt = *supported_rf_interface++; + if (supported_rf_interface + rf_extension_cnt > skb_tail_pointer(skb)) + break; + + /* Only count the entry after full validation */ + rf_interface_idx++; supported_rf_interface += rf_extension_cnt; } + ndev->num_supported_rf_interfaces = rf_interface_idx; + ndev->max_logical_connections = rsp->max_logical_connections; ndev->max_routing_table_size = __le16_to_cpu(rsp->max_routing_table_size); diff --git a/net/phonet/pep.c b/net/phonet/pep.c index 31b29e3ca7bc61..e188c563ee5b0a 100644 --- a/net/phonet/pep.c +++ b/net/phonet/pep.c @@ -1117,7 +1117,7 @@ static int pep_getsockopt(struct sock *sk, int level, int optname, len = min_t(unsigned int, sizeof(int), len); if (put_user(len, optlen)) return -EFAULT; - if (put_user(val, (int __user *) optval)) + if (copy_to_user(optval, &val, len)) return -EFAULT; return 0; } diff --git a/net/rds/cong.c b/net/rds/cong.c index 3133b91f9e695a..f7634ce3ffc105 100644 --- a/net/rds/cong.c +++ b/net/rds/cong.c @@ -256,9 +256,9 @@ void rds_cong_map_updated(struct rds_cong_map *map, uint64_t portmask) map, &map->m_addr); rds_stats_inc(s_cong_update_received); atomic_inc(&rds_cong_generation); - if (waitqueue_active(&map->m_waitq)) + if (wq_has_sleeper(&map->m_waitq)) wake_up(&map->m_waitq); - if (waitqueue_active(&rds_poll_waitq)) + if (wq_has_sleeper(&rds_poll_waitq)) wake_up_all(&rds_poll_waitq); if (portmask && !list_empty(&rds_cong_monitor)) { diff --git a/net/sched/act_api.c b/net/sched/act_api.c index 600b7804befd27..b4415d358c9114 100644 --- a/net/sched/act_api.c +++ b/net/sched/act_api.c @@ -452,7 +452,10 @@ static size_t tcf_action_shared_attrs_size(const struct tc_action *act) /* TCA_STATS_QUEUE */ + nla_total_size_64bit(sizeof(struct gnet_stats_queue)) + nla_total_size(0) /* TCA_ACT_OPTIONS nested */ - + nla_total_size(sizeof(struct tcf_t)); /* TCA_GACT_TM */ + /* TCA_GACT_TM; actions dump their tcf_t with nla_put_64bit(), + * which may emit an extra NLA_PAD attribute. + */ + + nla_total_size_64bit(sizeof(struct tcf_t)); } static size_t tcf_action_full_attrs_size(size_t sz) diff --git a/net/sched/act_bpf.c b/net/sched/act_bpf.c index 09d46e195e33b9..06d8f78b73683a 100644 --- a/net/sched/act_bpf.c +++ b/net/sched/act_bpf.c @@ -389,6 +389,31 @@ static void tcf_bpf_cleanup(struct tc_action *act) tcf_bpf_cfg_cleanup(&tmp); } +static size_t tcf_bpf_get_fill_size(const struct tc_action *act) +{ + struct tcf_bpf *prog = to_bpf(act); + size_t size = nla_total_size(sizeof(struct tc_act_bpf)); + + /* bpf_ops and bpf_num_ops are published as separate stores under + * tcf_lock, so take it here as tcf_bpf_dump() does. + */ + spin_lock_bh(&prog->tcf_lock); + if (tcf_bpf_is_ebpf(prog)) { + /* TCA_ACT_BPF_NAME */ + size += nla_total_size(ACT_BPF_NAME_LEN + 1); + size += nla_total_size(sizeof(u32)); /* TCA_ACT_BPF_ID */ + size += nla_total_size(BPF_TAG_SIZE); /* TCA_ACT_BPF_TAG */ + } else { + size += nla_total_size(sizeof(u16)); /* TCA_ACT_BPF_OPS_LEN */ + /* TCA_ACT_BPF_OPS */ + size += nla_total_size(prog->bpf_num_ops * + sizeof(struct sock_filter)); + } + spin_unlock_bh(&prog->tcf_lock); + + return size; +} + static struct tc_action_ops act_bpf_ops __read_mostly = { .kind = "bpf", .id = TCA_ID_BPF, @@ -397,6 +422,7 @@ static struct tc_action_ops act_bpf_ops __read_mostly = { .dump = tcf_bpf_dump, .cleanup = tcf_bpf_cleanup, .init = tcf_bpf_init, + .get_fill_size = tcf_bpf_get_fill_size, .size = sizeof(struct tcf_bpf), }; MODULE_ALIAS_NET_ACT("bpf"); diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index e250969c84aca4..370085ab6ea41e 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -1657,6 +1657,51 @@ static int tcf_ct_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_ct_get_fill_size(const struct tc_action *act) +{ + const struct tcf_ct_params *p; + size_t size; + + size = nla_total_size(sizeof(struct tc_ct)) /* TCA_CT_PARMS */ + + nla_total_size(sizeof(u16)); /* TCA_CT_ACTION */ + + rcu_read_lock(); + p = rcu_dereference(to_ct(act)->params); + + if (p->ct_action & TCA_CT_ACT_CLEAR) + goto out; + + /* TCA_CT_MARK, TCA_CT_MARK_MASK */ + if (IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)) + size += nla_total_size(sizeof(p->mark)) + + nla_total_size(sizeof(p->mark_mask)); + + /* TCA_CT_LABELS, TCA_CT_LABELS_MASK */ + if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS)) + size += nla_total_size(sizeof(p->labels)) + + nla_total_size(sizeof(p->labels_mask)); + + if (IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)) + size += nla_total_size(sizeof(p->zone)); /* TCA_CT_ZONE */ + + if (p->ct_action & TCA_CT_ACT_NAT) + /* TCA_CT_NAT_IPV6_{MIN,MAX}, the larger of the two address + * variants, plus TCA_CT_NAT_PORT_{MIN,MAX}. + */ + size += 2 * nla_total_size(sizeof(struct in6_addr)) + + 2 * nla_total_size(sizeof(__be16)); + + /* TCA_CT_HELPER_{NAME,FAMILY,PROTO} */ + if (p->helper) + size += nla_total_size(NF_CT_HELPER_NAME_LEN) + + nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(u8)); +out: + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_ct_ops = { .kind = "ct", .id = TCA_ID_CT, @@ -1666,6 +1711,7 @@ static struct tc_action_ops act_ct_ops = { .init = tcf_ct_init, .cleanup = tcf_ct_cleanup, .stats_update = tcf_stats_update, + .get_fill_size = tcf_ct_get_fill_size, .offload_act_setup = tcf_ct_offload_act_setup, .size = sizeof(struct tcf_ct), }; diff --git a/net/sched/act_ctinfo.c b/net/sched/act_ctinfo.c index 1886ffd2ca9569..fced4b1094af88 100644 --- a/net/sched/act_ctinfo.c +++ b/net/sched/act_ctinfo.c @@ -356,6 +356,16 @@ static void tcf_ctinfo_cleanup(struct tc_action *a) kfree_rcu(cp, rcu); } +static size_t tcf_ctinfo_get_fill_size(const struct tc_action *act) +{ + return nla_total_size(sizeof(struct tc_ctinfo)) /* TCA_CTINFO_ACT */ + + nla_total_size(sizeof(u16)) /* TCA_CTINFO_ZONE */ + /* TCA_CTINFO_PARMS_{DSCP_MASK,DSCP_STATEMASK,CPMARK_MASK} */ + + 3 * nla_total_size(sizeof(u32)) + /* TCA_CTINFO_STATS_{DSCP_SET,DSCP_ERROR,CPMARK_SET} */ + + 3 * nla_total_size_64bit(sizeof(u64)); +} + static struct tc_action_ops act_ctinfo_ops = { .kind = "ctinfo", .id = TCA_ID_CTINFO, @@ -364,6 +374,7 @@ static struct tc_action_ops act_ctinfo_ops = { .dump = tcf_ctinfo_dump, .init = tcf_ctinfo_init, .cleanup= tcf_ctinfo_cleanup, + .get_fill_size = tcf_ctinfo_get_fill_size, .size = sizeof(struct tcf_ctinfo), }; MODULE_ALIAS_NET_ACT("ctinfo"); diff --git a/net/sched/act_ife.c b/net/sched/act_ife.c index 065228026c58eb..9cea71fc1db3d4 100644 --- a/net/sched/act_ife.c +++ b/net/sched/act_ife.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -723,7 +724,7 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, tcf_lastuse_update(&ife->tcf_tm); if (skb_at_tc_ingress(skb)) - skb_push(skb, skb->dev->hard_header_len); + skb_push(skb, ETH_HLEN); tlv_data = ife_decode(skb, &metalen); if (unlikely(!tlv_data)) { @@ -795,7 +796,7 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a, where ORIGDATA = original ethernet header ... */ u16 metalen = ife_get_sz(skb, p); - int hdrm = metalen + skb->dev->hard_header_len + IFE_METAHDRLEN; + int hdrm = metalen + ETH_HLEN + IFE_METAHDRLEN; unsigned int skboff = 0; int new_len = skb->len + hdrm; bool exceed_mtu = false; @@ -826,7 +827,7 @@ drop: } if (skb_at_tc_ingress(skb)) - skb_push(skb, skb->dev->hard_header_len); + skb_push(skb, ETH_HLEN); ife_meta = ife_encode(skb, metalen); if (!ife_meta) @@ -856,11 +857,27 @@ drop: oethh->h_proto = htons(p->eth_type); if (skb_at_tc_ingress(skb)) - skb_pull(skb, skb->dev->hard_header_len); + skb_pull(skb, ETH_HLEN); return action; } +/* IFE encapsulates the original Ethernet header and, on decode, expects to + * find one, so it can only ever work on skbs that carry one. Loopback carries + * Ethernet header as well, so it qualifies here. + * At ingress, also verify that the L2 header about to be pushed back really + * is an Ethernet header because the skb could've been redirected with mirred + * from a non-Ethernet device. + */ +static bool tcf_ife_is_eth_skb(const struct sk_buff *skb) +{ + if (skb->dev->type != ARPHRD_ETHER && + skb->dev->type != ARPHRD_LOOPBACK) + return false; + + return !skb_at_tc_ingress(skb) || skb->mac_len == ETH_HLEN; +} + TC_INDIRECT_SCOPE int tcf_ife_act(struct sk_buff *skb, const struct tc_action *a, struct tcf_result *res) @@ -869,6 +886,13 @@ TC_INDIRECT_SCOPE int tcf_ife_act(struct sk_buff *skb, struct tcf_ife_params *p; int ret; + if (unlikely(!tcf_ife_is_eth_skb(skb))) { + bstats_update(this_cpu_ptr(ife->common.cpu_bstats), skb); + tcf_lastuse_update(&ife->tcf_tm); + qstats_cpu_drop_inc(ife->common.cpu_qstats); + return TC_ACT_SHOT; + } + p = rcu_dereference_bh(ife->params); if (p->flags & IFE_ENCODE) { ret = tcf_ife_encode(skb, a, res, p); @@ -878,6 +902,28 @@ TC_INDIRECT_SCOPE int tcf_ife_act(struct sk_buff *skb, return tcf_ife_decode(skb, a, res); } +static size_t tcf_ife_get_fill_size(const struct tc_action *act) +{ + struct tcf_ife_info *ife = to_ife(act); + const struct tcf_ife_params *p; + struct tcf_meta_info *e; + size_t size = nla_total_size(sizeof(struct tc_ife)) /* TCA_IFE_PARMS */ + + nla_total_size(ETH_ALEN) /* TCA_IFE_DMAC */ + + nla_total_size(ETH_ALEN) /* TCA_IFE_SMAC */ + + nla_total_size(2) /* TCA_IFE_TYPE */ + + nla_total_size(0); /* TCA_IFE_METALST */ + + rcu_read_lock(); + p = rcu_dereference(ife->params); + if (p) { + list_for_each_entry_rcu(e, &p->metalist, metalist) + size += nla_total_size(sizeof(u32)); + } + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_ife_ops = { .kind = "ife", .id = TCA_ID_IFE, @@ -886,6 +932,7 @@ static struct tc_action_ops act_ife_ops = { .dump = tcf_ife_dump, .cleanup = tcf_ife_cleanup, .init = tcf_ife_init, + .get_fill_size = tcf_ife_get_fill_size, .size = sizeof(struct tcf_ife_info), }; MODULE_ALIAS_NET_ACT("ife"); diff --git a/net/sched/act_pedit.c b/net/sched/act_pedit.c index d4d47a9921f457..99d7e36510bd08 100644 --- a/net/sched/act_pedit.c +++ b/net/sched/act_pedit.c @@ -626,6 +626,29 @@ static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_pedit_get_fill_size(const struct tc_action *act) +{ + const struct tcf_pedit_parms *parms; + size_t size; + + rcu_read_lock(); + parms = rcu_dereference(to_pedit(act)->parms); + size = nla_total_size(struct_size_t(struct tc_pedit, keys, + parms->tcfp_nkeys)); + if (parms->tcfp_keys_ex) { + /* TCA_PEDIT_KEYS_EX, holding one TCA_PEDIT_KEY_EX nest with a + * HTYPE and a CMD attribute per key. + */ + size += nla_total_size(0) + + parms->tcfp_nkeys * (nla_total_size(0) + + nla_total_size(sizeof(u16)) + + nla_total_size(sizeof(u16))); + } + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_pedit_ops = { .kind = "pedit", .id = TCA_ID_PEDIT, @@ -635,6 +658,7 @@ static struct tc_action_ops act_pedit_ops = { .dump = tcf_pedit_dump, .cleanup = tcf_pedit_cleanup, .init = tcf_pedit_init, + .get_fill_size = tcf_pedit_get_fill_size, .offload_act_setup = tcf_pedit_offload_act_setup, .size = sizeof(struct tcf_pedit), }; diff --git a/net/sched/act_police.c b/net/sched/act_police.c index ce08f6840ef7ca..3f8147f3754933 100644 --- a/net/sched/act_police.c +++ b/net/sched/act_police.c @@ -490,6 +490,17 @@ static int tcf_police_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_police_get_fill_size(const struct tc_action *act) +{ + return nla_total_size(sizeof(struct tc_police)) /* TCA_POLICE_TBF */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_RATE64 */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_PEAKRATE64 */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_PKTRATE64 */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_PKTBURST64 */ + + nla_total_size(sizeof(u32)) /* TCA_POLICE_RESULT */ + + nla_total_size(sizeof(u32)); /* TCA_POLICE_AVRATE */ +} + MODULE_AUTHOR("Alexey Kuznetsov"); MODULE_DESCRIPTION("Policing actions"); MODULE_LICENSE("GPL"); @@ -503,6 +514,7 @@ static struct tc_action_ops act_police_ops = { .dump = tcf_police_dump, .init = tcf_police_init, .cleanup = tcf_police_cleanup, + .get_fill_size = tcf_police_get_fill_size, .offload_act_setup = tcf_police_offload_act_setup, .size = sizeof(struct tcf_police), }; diff --git a/net/sched/act_sample.c b/net/sched/act_sample.c index 2ceb4d141b713d..44319a159b55d5 100644 --- a/net/sched/act_sample.c +++ b/net/sched/act_sample.c @@ -315,6 +315,14 @@ static int tcf_sample_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_sample_get_fill_size(const struct tc_action *act) +{ + return nla_total_size(sizeof(struct tc_sample)) /* TCA_SAMPLE_PARMS */ + + nla_total_size(sizeof(u32)) /* TCA_SAMPLE_RATE */ + + nla_total_size(sizeof(u32)) /* TCA_SAMPLE_TRUNC_SIZE */ + + nla_total_size(sizeof(u32)); /* TCA_SAMPLE_PSAMPLE_GROUP */ +} + static struct tc_action_ops act_sample_ops = { .kind = "sample", .id = TCA_ID_SAMPLE, @@ -324,6 +332,7 @@ static struct tc_action_ops act_sample_ops = { .dump = tcf_sample_dump, .init = tcf_sample_init, .cleanup = tcf_sample_cleanup, + .get_fill_size = tcf_sample_get_fill_size, .get_psample_group = tcf_sample_get_group, .offload_act_setup = tcf_sample_offload_act_setup, .size = sizeof(struct tcf_sample), diff --git a/net/sched/act_skbmod.c b/net/sched/act_skbmod.c index a464b0a3c1b81d..7579cf1e0ff37c 100644 --- a/net/sched/act_skbmod.c +++ b/net/sched/act_skbmod.c @@ -38,7 +38,6 @@ TC_INDIRECT_SCOPE int tcf_skbmod_act(struct sk_buff *skb, if (unlikely(p->action == TC_ACT_SHOT)) goto drop; - max_edit_len = skb_mac_header_len(skb); flags = p->flags; /* tcf_skbmod_init() guarantees "flags" to be one of the following: @@ -51,14 +50,19 @@ TC_INDIRECT_SCOPE int tcf_skbmod_act(struct sk_buff *skb, if (flags == SKBMOD_F_ECN) { switch (skb_protocol(skb, true)) { case cpu_to_be16(ETH_P_IP): + max_edit_len = sizeof(struct iphdr); + break; case cpu_to_be16(ETH_P_IPV6): - max_edit_len += skb_network_header_len(skb); + max_edit_len = sizeof(struct ipv6hdr); break; default: goto out; } - } else if (!skb->dev || skb->dev->type != ARPHRD_ETHER) { - goto out; + max_edit_len += skb_network_offset(skb); + } else { + if (!skb->dev || skb->dev->type != ARPHRD_ETHER) + goto out; + max_edit_len = ETH_HLEN; } err = skb_ensure_writable(skb, max_edit_len); diff --git a/net/sched/act_tunnel_key.c b/net/sched/act_tunnel_key.c index b14807761d829e..ff401ace4f3dae 100644 --- a/net/sched/act_tunnel_key.c +++ b/net/sched/act_tunnel_key.c @@ -835,6 +835,85 @@ static int tcf_tunnel_key_offload_act_setup(struct tc_action *act, return 0; } +static size_t +tunnel_key_geneve_opts_fill_size(const struct ip_tunnel_info *info) +{ + const u8 *src = ip_tunnel_info_opts(info); + int len = info->options_len; + size_t size = 0; + + while (len > 0) { + const struct geneve_opt *opt = (const struct geneve_opt *)src; + + /* TCA_TUNNEL_KEY_ENC_OPT_GENEVE_{CLASS,TYPE,DATA} */ + size += nla_total_size(2) + + nla_total_size(1) + + nla_total_size(opt->length * 4); + + len -= sizeof(struct geneve_opt) + opt->length * 4; + src += sizeof(struct geneve_opt) + opt->length * 4; + } + + return size; +} + +static size_t tunnel_key_opts_fill_size(const struct ip_tunnel_info *info) +{ + size_t size; + + if (!info->options_len) + return 0; + + /* TCA_TUNNEL_KEY_ENC_OPTS and the per-protocol nest inside it */ + size = nla_total_size(0) + nla_total_size(0); + + if (test_bit(IP_TUNNEL_GENEVE_OPT_BIT, info->key.tun_flags)) { + size += tunnel_key_geneve_opts_fill_size(info); + } else if (test_bit(IP_TUNNEL_VXLAN_OPT_BIT, info->key.tun_flags)) { + /* TCA_TUNNEL_KEY_ENC_OPT_VXLAN_GBP */ + size += nla_total_size(sizeof(u32)); + } else if (test_bit(IP_TUNNEL_ERSPAN_OPT_BIT, info->key.tun_flags)) { + /* TCA_TUNNEL_KEY_ENC_OPT_ERSPAN_{VER,INDEX,DIR,HWID} */ + size += nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(__be32)) + + nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(u8)); + } + + return size; +} + +static size_t tunnel_key_get_fill_size(const struct tc_action *act) +{ + struct tcf_tunnel_key *t = to_tunnel_key(act); + const struct tcf_tunnel_key_params *params; + /* TCA_TUNNEL_KEY_PARMS */ + size_t size = nla_total_size(sizeof(struct tc_tunnel_key)); + + rcu_read_lock(); + params = rcu_dereference(t->params); + if (params->tcft_action == TCA_TUNNEL_KEY_ACT_SET) { + const struct ip_tunnel_info *info = + ¶ms->tcft_enc_metadata->u.tun_info; + + /* In dump order: TCA_TUNNEL_KEY_ENC_KEY_ID, the IPv6 address + * pair (larger than the IPv4 one), ..._ENC_DST_PORT, + * ..._NO_CSUM, ..._NO_FRAG, the options and ..._ENC_{TOS,TTL}. + */ + size += nla_total_size(sizeof(__be32)) + + 2 * nla_total_size(sizeof(struct in6_addr)) + + nla_total_size(sizeof(__be16)) + + nla_total_size(sizeof(u8)) + + nla_total_size(0) + + tunnel_key_opts_fill_size(info) + + nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(u8)); + } + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_tunnel_key_ops = { .kind = "tunnel_key", .id = TCA_ID_TUNNEL_KEY, @@ -843,6 +922,7 @@ static struct tc_action_ops act_tunnel_key_ops = { .dump = tunnel_key_dump, .init = tunnel_key_init, .cleanup = tunnel_key_release, + .get_fill_size = tunnel_key_get_fill_size, .offload_act_setup = tcf_tunnel_key_offload_act_setup, .size = sizeof(struct tcf_tunnel_key), }; diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index 4e6a2812a4f325..9966766661d50d 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -2248,6 +2248,12 @@ static bool is_ingress_or_clsact(struct tcf_block *block, struct Qdisc *q) return tcf_block_shared(block) || (q && !!(q->flags & TCQ_F_INGRESS)); } +enum tcf_tp_insert_state { + TP_NOT_CREATED = 0, /* did not create and insert a new tp */ + TP_CREATED, /* created and inserted a new tp */ + TP_NOT_OWNED, /* created a proto but failed to insert */ +}; + static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, struct netlink_ext_ack *extack) { @@ -2268,12 +2274,12 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, unsigned long cl; void *fh; int err; - int tp_created; + enum tcf_tp_insert_state tp_state; bool rtnl_held = false; u32 flags; replay: - tp_created = 0; + tp_state = TP_NOT_CREATED; err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack); @@ -2395,13 +2401,15 @@ replay: goto errout_tp; } - tp_created = 1; + tp_state = TP_CREATED; tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio, rtnl_held); if (IS_ERR(tp)) { err = PTR_ERR(tp); goto errout_tp; } + if (tp != tp_new) + tp_state = TP_NOT_OWNED; } else { mutex_unlock(&chain->filter_chain_lock); } @@ -2455,13 +2463,13 @@ replay: } errout: - if (err && tp_created) + if (err && tp_state == TP_CREATED) tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL); errout_tp: if (chain) { if (tp && !IS_ERR(tp)) tcf_proto_put(tp, rtnl_held, NULL); - if (!tp_created) + if (tp_state == TP_NOT_CREATED) tcf_chain_put(chain); } tcf_block_release(q, block, rtnl_held); @@ -3364,7 +3372,8 @@ int tcf_exts_init_ex(struct tcf_exts *exts, struct net *net, int action, * This reference might be taken later from tcf_exts_get_net(). */ exts->net = net; - exts->actions = kzalloc_objs(struct tc_action *, TCA_ACT_MAX_PRIO); + exts->actions = kzalloc_objs(struct tc_action *, TCA_ACT_MAX_PRIO, + GFP_KERNEL_ACCOUNT); if (!exts->actions) return -ENOMEM; #endif diff --git a/net/sched/cls_basic.c b/net/sched/cls_basic.c index 492cd9ce8d46ff..e2a94ba9fba76a 100644 --- a/net/sched/cls_basic.c +++ b/net/sched/cls_basic.c @@ -193,7 +193,7 @@ static int basic_change(struct net *net, struct sk_buff *in_skb, return -EINVAL; } - fnew = kzalloc_obj(*fnew); + fnew = kzalloc_obj(*fnew, GFP_KERNEL_ACCOUNT); if (!fnew) return -ENOBUFS; @@ -212,9 +212,11 @@ static int basic_change(struct net *net, struct sk_buff *in_skb, if (err) goto errout; fnew->handle = handle; - fnew->pf = alloc_percpu(struct tc_basic_pcnt); + fnew->pf = alloc_percpu_gfp(struct tc_basic_pcnt, GFP_KERNEL_ACCOUNT); if (!fnew->pf) { err = -ENOMEM; + if (!fold) + idr_remove(&head->handle_idr, fnew->handle); goto errout; } diff --git a/net/sched/cls_bpf.c b/net/sched/cls_bpf.c index 6d19155becc86a..188cf0f949dd48 100644 --- a/net/sched/cls_bpf.c +++ b/net/sched/cls_bpf.c @@ -352,7 +352,7 @@ static int cls_bpf_prog_from_ops(struct nlattr **tb, struct cls_bpf_prog *prog) if (bpf_size != nla_len(tb[TCA_BPF_OPS])) return -EINVAL; - bpf_ops = kmemdup(nla_data(tb[TCA_BPF_OPS]), bpf_size, GFP_KERNEL); + bpf_ops = kmemdup(nla_data(tb[TCA_BPF_OPS]), bpf_size, GFP_KERNEL_ACCOUNT); if (bpf_ops == NULL) return -ENOMEM; @@ -403,7 +403,7 @@ static int cls_bpf_prog_from_efd(struct nlattr **tb, struct cls_bpf_prog *prog, } if (tb[TCA_BPF_NAME]) { - name = nla_memdup(tb[TCA_BPF_NAME], GFP_KERNEL); + name = nla_memdup(tb[TCA_BPF_NAME], GFP_KERNEL_ACCOUNT); if (!name) { bpf_prog_put(fp); return -ENOMEM; @@ -443,7 +443,7 @@ static int cls_bpf_change(struct net *net, struct sk_buff *in_skb, if (ret < 0) return ret; - prog = kzalloc_obj(*prog); + prog = kzalloc_obj(*prog, GFP_KERNEL_ACCOUNT); if (!prog) return -ENOBUFS; diff --git a/net/sched/cls_cgroup.c b/net/sched/cls_cgroup.c index 680a5c308094e1..210fd9fd26d834 100644 --- a/net/sched/cls_cgroup.c +++ b/net/sched/cls_cgroup.c @@ -95,7 +95,7 @@ static int cls_cgroup_change(struct net *net, struct sk_buff *in_skb, if (head && handle != head->handle) return -ENOENT; - new = kzalloc_obj(*head); + new = kzalloc_obj(*head, GFP_KERNEL_ACCOUNT); if (!new) return -ENOBUFS; diff --git a/net/sched/cls_flow.c b/net/sched/cls_flow.c index 356c68ebc3895a..a9ac3acf6eda04 100644 --- a/net/sched/cls_flow.c +++ b/net/sched/cls_flow.c @@ -438,7 +438,7 @@ static int flow_change(struct net *net, struct sk_buff *in_skb, return -EOPNOTSUPP; } - fnew = kzalloc_obj(*fnew); + fnew = kzalloc_obj(*fnew, GFP_KERNEL_ACCOUNT); if (!fnew) return -ENOBUFS; diff --git a/net/sched/cls_flower.c b/net/sched/cls_flower.c index 88f8a32fab2b40..0e275b58151c4a 100644 --- a/net/sched/cls_flower.c +++ b/net/sched/cls_flower.c @@ -2233,7 +2233,7 @@ static struct fl_flow_mask *fl_create_new_mask(struct cls_fl_head *head, struct fl_flow_mask *newmask; int err; - newmask = kzalloc_obj(*newmask); + newmask = kzalloc_obj(*newmask, GFP_KERNEL_ACCOUNT); if (!newmask) return ERR_PTR(-ENOMEM); @@ -2394,7 +2394,7 @@ static int fl_change(struct net *net, struct sk_buff *in_skb, goto errout_tb; } - fnew = kzalloc_obj(*fnew); + fnew = kzalloc_obj(*fnew, GFP_KERNEL_ACCOUNT); if (!fnew) { err = -ENOBUFS; goto errout_tb; diff --git a/net/sched/cls_fw.c b/net/sched/cls_fw.c index 646a730dca93c9..a462b262719c30 100644 --- a/net/sched/cls_fw.c +++ b/net/sched/cls_fw.c @@ -276,7 +276,7 @@ static int fw_change(struct net *net, struct sk_buff *in_skb, if (f->id != handle && handle) return -EINVAL; - fnew = kzalloc_obj(struct fw_filter); + fnew = kzalloc_obj(struct fw_filter, GFP_KERNEL_ACCOUNT); if (!fnew) return -ENOBUFS; @@ -330,7 +330,7 @@ static int fw_change(struct net *net, struct sk_buff *in_skb, rcu_assign_pointer(tp->root, head); } - f = kzalloc_obj(struct fw_filter); + f = kzalloc_obj(struct fw_filter, GFP_KERNEL_ACCOUNT); if (f == NULL) return -ENOBUFS; diff --git a/net/sched/cls_matchall.c b/net/sched/cls_matchall.c index 6f126872c14a08..c14899b935bfdc 100644 --- a/net/sched/cls_matchall.c +++ b/net/sched/cls_matchall.c @@ -189,7 +189,7 @@ static int mall_change(struct net *net, struct sk_buff *in_skb, return -EINVAL; } - new = kzalloc_obj(*new); + new = kzalloc_obj(*new, GFP_KERNEL_ACCOUNT); if (!new) return -ENOBUFS; @@ -201,7 +201,7 @@ static int mall_change(struct net *net, struct sk_buff *in_skb, handle = 1; new->handle = handle; new->flags = userflags; - new->pf = alloc_percpu(struct tc_matchall_pcnt); + new->pf = alloc_percpu_gfp(struct tc_matchall_pcnt, GFP_KERNEL_ACCOUNT); if (!new->pf) { err = -ENOMEM; goto err_alloc_percpu; diff --git a/net/sched/cls_route.c b/net/sched/cls_route.c index eded7aacd3f70e..0d1324c9058373 100644 --- a/net/sched/cls_route.c +++ b/net/sched/cls_route.c @@ -455,7 +455,7 @@ static int route4_set_parms(struct net *net, struct tcf_proto *tp, h1 = to_hash(nhandle); b = rtnl_dereference(head->table[h1]); if (!b) { - b = kzalloc_obj(struct route4_bucket); + b = kzalloc_obj(struct route4_bucket, GFP_KERNEL_ACCOUNT); if (b == NULL) return -ENOBUFS; @@ -524,7 +524,7 @@ static int route4_change(struct net *net, struct sk_buff *in_skb, return -EINVAL; err = -ENOBUFS; - f = kzalloc_obj(struct route4_filter); + f = kzalloc_obj(struct route4_filter, GFP_KERNEL_ACCOUNT); if (!f) goto errout; diff --git a/net/sched/cls_u32.c b/net/sched/cls_u32.c index c297d7dbcf9163..ac6d0fa5a40e3f 100644 --- a/net/sched/cls_u32.c +++ b/net/sched/cls_u32.c @@ -825,7 +825,7 @@ static struct tc_u_knode *u32_init_knode(struct net *net, struct tcf_proto *tp, struct tc_u32_sel *s = &n->sel; struct tc_u_knode *new; - new = kzalloc_flex(*new, sel.keys, s->nkeys); + new = kzalloc_flex(*new, sel.keys, s->nkeys, GFP_KERNEL_ACCOUNT); if (!new) return NULL; @@ -1114,15 +1114,16 @@ static int u32_change(struct net *net, struct sk_buff *in_skb, goto erridr; } - n = kzalloc_flex(*n, sel.keys, s->nkeys); + n = kzalloc_flex(*n, sel.keys, s->nkeys, GFP_KERNEL_ACCOUNT); if (n == NULL) { err = -ENOBUFS; goto erridr; } #ifdef CONFIG_CLS_U32_PERF - n->pf = __alloc_percpu(struct_size(n->pf, kcnts, s->nkeys), - __alignof__(struct tc_u32_pcnt)); + n->pf = __alloc_percpu_gfp(struct_size(n->pf, kcnts, s->nkeys), + __alignof__(struct tc_u32_pcnt), + GFP_KERNEL_ACCOUNT); if (!n->pf) { err = -ENOBUFS; goto errfree; @@ -1144,7 +1145,7 @@ static int u32_change(struct net *net, struct sk_buff *in_skb, goto errout; #ifdef CONFIG_CLS_U32_MARK - n->pcpu_success = alloc_percpu(u32); + n->pcpu_success = alloc_percpu_gfp(u32, GFP_KERNEL_ACCOUNT); if (!n->pcpu_success) { err = -ENOMEM; goto errout; diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index 65b35528d125f4..90503e59e6e3ba 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -610,8 +610,11 @@ void __qdisc_calculate_pkt_len(struct sk_buff *skb, pkt_len <<= stab->szopts.size_log; out: - if (unlikely(pkt_len < 1)) - pkt_len = 1; + /* A size table can inflate qdisc_pkt_len() beyond any real packet + * (via overhead, the data table, or size_log); cap it so deficit + * schedulers such as DRR/ETS terminate their refill loops. + */ + pkt_len = clamp_t(int, pkt_len, 1, QDISC_PKT_LEN_MAX); qdisc_skb_cb(skb)->pkt_len = pkt_len; } diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c index f25f60978631a4..dc93267029e709 100644 --- a/net/sched/sch_cake.c +++ b/net/sched/sch_cake.c @@ -1907,6 +1907,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, ktime_add_ms(q->last_reconfig_time, 250))) { q->config->rate_bps = (q->avg_peak_bandwidth * 15) >> 4; + q->last_reconfig_time = now; cake_reconfigure(sch); } } diff --git a/net/sched/sch_codel.c b/net/sched/sch_codel.c index cacf5244958e8f..6aa5829d696124 100644 --- a/net/sched/sch_codel.c +++ b/net/sched/sch_codel.c @@ -205,7 +205,7 @@ static int codel_init(struct Qdisc *sch, struct nlattr *opt, codel_params_init(&q->params); codel_vars_init(&q->vars); codel_stats_init(&q->stats); - q->params.mtu = psched_mtu(qdisc_dev(sch)); + q->params.mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 256, 1 << 20); if (opt) { int err = codel_change(sch, opt, extack); diff --git a/net/sched/sch_fifo.c b/net/sched/sch_fifo.c index e6bfd39ff33965..1b6388d50967f6 100644 --- a/net/sched/sch_fifo.c +++ b/net/sched/sch_fifo.c @@ -19,7 +19,7 @@ static int bfifo_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) { - if (likely(sch->qstats.backlog + qdisc_pkt_len(skb) <= + if (likely((u64)sch->qstats.backlog + qdisc_pkt_len(skb) <= READ_ONCE(sch->limit))) return qdisc_enqueue_tail(skb, sch); diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c index 7cae082a984721..35f940b2205df1 100644 --- a/net/sched/sch_fq.c +++ b/net/sched/sch_fq.c @@ -828,8 +828,12 @@ begin: * f->time_next_packet was set when prior packet was sent, * and current time (@now) can be too late by tens of us. */ - if (f->time_next_packet) - len -= min(len/2, now - f->time_next_packet); + if (f->time_next_packet) { + s64 drift = now - f->time_next_packet; + + if (drift > 0) + len -= min_t(u64, len / 2, drift); + } f->time_next_packet = now + len; } out: @@ -976,7 +980,7 @@ static int fq_resize(struct Qdisc *sch, u32 log) } static const struct netlink_range_validation iq_range = { - .max = INT_MAX, + .max = 1 << 20, }; static const struct nla_policy fq_policy[TCA_FQ_MAX + 1] = { @@ -1102,14 +1106,10 @@ static int fq_change(struct Qdisc *sch, struct nlattr *opt, nla_get_u32(tb[TCA_FQ_FLOW_PLIMIT])); if (tb[TCA_FQ_QUANTUM]) { - u32 quantum = nla_get_u32(tb[TCA_FQ_QUANTUM]); + u32 quantum = clamp_t(u32, nla_get_u32(tb[TCA_FQ_QUANTUM]), + 256, 1 << 20); - if (quantum > 0 && quantum <= (1 << 20)) { - WRITE_ONCE(q->quantum, quantum); - } else { - NL_SET_ERR_MSG_MOD(extack, "invalid quantum"); - err = -EINVAL; - } + WRITE_ONCE(q->quantum, quantum); } if (tb[TCA_FQ_INITIAL_QUANTUM]) @@ -1222,12 +1222,14 @@ static int fq_init(struct Qdisc *sch, struct nlattr *opt, struct netlink_ext_ack *extack) { struct fq_sched_data *q = qdisc_priv(sch); + u32 mtu; int i, err; sch->limit = 10000; q->flow_plimit = 100; - q->quantum = 2 * psched_mtu(qdisc_dev(sch)); - q->initial_quantum = 10 * psched_mtu(qdisc_dev(sch)); + mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 1, 1 << 20); + q->quantum = clamp_t(u32, 2 * mtu, 256, 1 << 20); + q->initial_quantum = min_t(u32, 10 * mtu, 1 << 20); q->flow_refill_delay = msecs_to_jiffies(40); q->flow_max_rate = ~0UL; q->time_next_delayed_flow = ~0ULL; diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c index 6cce86ba383cf6..969b2510b0b86e 100644 --- a/net/sched/sch_fq_codel.c +++ b/net/sched/sch_fq_codel.c @@ -509,6 +509,7 @@ static int fq_codel_init(struct Qdisc *sch, struct nlattr *opt, struct netlink_ext_ack *extack) { struct fq_codel_sched_data *q = qdisc_priv(sch); + u32 mtu; int i; int err; @@ -516,13 +517,14 @@ static int fq_codel_init(struct Qdisc *sch, struct nlattr *opt, q->flows_cnt = 1024; q->memory_limit = 32 << 20; /* 32 MBytes */ q->drop_batch_size = 64; - q->quantum = psched_mtu(qdisc_dev(sch)); + mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 256, FQ_CODEL_QUANTUM_MAX); + q->quantum = mtu; INIT_LIST_HEAD(&q->new_flows); INIT_LIST_HEAD(&q->old_flows); codel_params_init(&q->cparams); codel_stats_init(&q->cstats); q->cparams.ecn = true; - q->cparams.mtu = psched_mtu(qdisc_dev(sch)); + q->cparams.mtu = mtu; if (opt) { err = fq_codel_change(sch, opt, extack); diff --git a/net/sched/sch_fq_pie.c b/net/sched/sch_fq_pie.c index 069e1facd4130d..b27d95418707a5 100644 --- a/net/sched/sch_fq_pie.c +++ b/net/sched/sch_fq_pie.c @@ -427,7 +427,8 @@ static int fq_pie_init(struct Qdisc *sch, struct nlattr *opt, pie_params_init(&q->p_params); sch->limit = 10 * 1024; q->p_params.limit = sch->limit; - q->quantum = psched_mtu(qdisc_dev(sch)); + q->quantum = clamp_t(u32, psched_mtu(qdisc_dev(sch)), + 256, 1 << 20); q->sch = sch; q->ecn_prob = 10; q->flows_cnt = 1024; diff --git a/net/sched/sch_gred.c b/net/sched/sch_gred.c index fcc1a4c0363624..f04f425c6c4495 100644 --- a/net/sched/sch_gred.c +++ b/net/sched/sch_gred.c @@ -179,7 +179,7 @@ static int gred_enqueue(struct sk_buff *skb, struct Qdisc *sch, * if no default DP has been configured. This * allows for DP flows to be left untouched. */ - if (likely(sch->qstats.backlog + qdisc_pkt_len(skb) <= + if (likely((u64)sch->qstats.backlog + qdisc_pkt_len(skb) <= sch->limit)) return qdisc_enqueue_tail(skb, sch); else @@ -244,7 +244,7 @@ static int gred_enqueue(struct sk_buff *skb, struct Qdisc *sch, break; } - if (gred_backlog(t, q, sch) + qdisc_pkt_len(skb) <= q->limit) { + if ((u64)gred_backlog(t, q, sch) + qdisc_pkt_len(skb) <= q->limit) { q->backlog += qdisc_pkt_len(skb); return qdisc_enqueue_tail(skb, sch); } diff --git a/net/sched/sch_hhf.c b/net/sched/sch_hhf.c index d85cb0263b675a..96acab6a8da035 100644 --- a/net/sched/sch_hhf.c +++ b/net/sched/sch_hhf.c @@ -624,6 +624,10 @@ static int hhf_init(struct Qdisc *sch, struct nlattr *opt, q->hhf_evict_timeout = HZ; /* 1 sec */ q->hhf_non_hh_weight = 2; + if ((int)q->quantum <= 0 || + (u64)q->quantum * q->hhf_non_hh_weight > INT_MAX) + q->quantum = 256; + if (opt) { int err = hhf_change(sch, opt, extack); diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c index fdac0dc8f35a37..1ba67b121de475 100644 --- a/net/sched/sch_htb.c +++ b/net/sched/sch_htb.c @@ -224,6 +224,7 @@ static struct htb_class *htb_classify(struct sk_buff *skb, struct Qdisc *sch, struct htb_class *cl; struct tcf_result res; struct tcf_proto *tcf; + unsigned int hops = 0; int result; /* allow to select class by setting skb->priority to valid classid; @@ -266,6 +267,10 @@ static struct htb_class *htb_classify(struct sk_buff *skb, struct Qdisc *sch, if (!cl->level) return cl; /* we hit leaf; return it */ + if (++hops > TC_HTB_MAXDEPTH) { + pr_warn_ratelimited("htb: classify loop detected, dropping packet\n"); + return NULL; + } /* we have got inner class; apply inner filter chain */ tcf = rcu_dereference_bh(cl->filter_list); } @@ -633,13 +638,11 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch, } else { return qdisc_drop(skb, sch, to_free); } -#ifdef CONFIG_NET_CLS_ACT } else if (!cl) { if (ret & __NET_XMIT_BYPASS) qdisc_qstats_drop(sch); __qdisc_drop(skb, to_free); return ret; -#endif } else if ((ret = qdisc_enqueue(skb, cl->leaf.q, to_free)) != NET_XMIT_SUCCESS) { if (net_xmit_drop_count(ret)) { diff --git a/net/sched/sch_plug.c b/net/sched/sch_plug.c index cefb65201e178c..b60ddfee6a68a2 100644 --- a/net/sched/sch_plug.c +++ b/net/sched/sch_plug.c @@ -89,7 +89,7 @@ static int plug_enqueue(struct sk_buff *skb, struct Qdisc *sch, { struct plug_sched_data *q = qdisc_priv(sch); - if (likely(sch->qstats.backlog + skb->len <= q->limit)) { + if (likely((u64)sch->qstats.backlog + skb->len <= q->limit)) { if (!q->unplug_indefinite) q->pkts_current_epoch++; return qdisc_enqueue_tail(skb, sch); diff --git a/net/sched/sch_sfq.c b/net/sched/sch_sfq.c index 77675f9a4c46e7..187d3ed578f268 100644 --- a/net/sched/sch_sfq.c +++ b/net/sched/sch_sfq.c @@ -799,7 +799,8 @@ static int sfq_init(struct Qdisc *sch, struct nlattr *opt, q->tail = NULL; q->divisor = SFQ_DEFAULT_HASH_DIVISOR; q->maxflows = SFQ_DEFAULT_FLOWS; - q->quantum = psched_mtu(qdisc_dev(sch)); + q->quantum = clamp_t(u32, psched_mtu(qdisc_dev(sch)), + 256, 1 << 20); q->perturb_period = 0; get_random_bytes(&q->perturbation, sizeof(q->perturbation)); diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c index 5c42a29a981cb6..9e52afc2d9808c 100644 --- a/net/sched/sch_teql.c +++ b/net/sched/sch_teql.c @@ -377,6 +377,7 @@ restart: nores = 1; break; } + skb->dev = dev; __skb_pull(skb, skb_network_offset(skb)); } while ((q = rcu_dereference(NEXT_SLAVE(q))) != start); diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c index cff910cedbfccb..e9f93b3ab435bb 100644 --- a/net/smc/af_smc.c +++ b/net/smc/af_smc.c @@ -409,13 +409,13 @@ void smc_sk_init(struct net *net, struct sock *sk, int protocol) "sk_lock-AF_SMC", &smc_key); spin_lock_init(&smc->accept_q_lock); spin_lock_init(&smc->conn.send_lock); - sk->sk_prot->hash(sk); mutex_init(&smc->clcsock_release_lock); smc_init_saved_callbacks(smc); smc->limit_smc_hs = net->smc.limit_smc_hs; smc->use_fallback = false; /* assume rdma capability first */ smc->fallback_rsn = 0; smc_close_init(smc); + sk->sk_prot->hash(sk); } static struct sock *smc_sock_alloc(struct net *net, struct socket *sock, diff --git a/net/smc/smc_inet.c b/net/smc/smc_inet.c index a94084b4a498ee..520b666fdd8fa2 100644 --- a/net/smc/smc_inet.c +++ b/net/smc/smc_inet.c @@ -15,13 +15,16 @@ #include "smc_inet.h" #include "smc.h" +#include "smc_close.h" static int smc_inet_init_sock(struct sock *sk); +static void smc_inet_destroy_sock(struct sock *sk); static struct proto smc_inet_prot = { .name = "INET_SMC", .owner = THIS_MODULE, .init = smc_inet_init_sock, + .destroy = smc_inet_destroy_sock, .hash = smc_hash_sk, .unhash = smc_unhash_sk, .release_cb = smc_release_cb, @@ -68,6 +71,7 @@ static struct proto smc_inet6_prot = { .name = "INET6_SMC", .owner = THIS_MODULE, .init = smc_inet_init_sock, + .destroy = smc_inet_destroy_sock, .hash = smc_hash_sk, .unhash = smc_unhash_sk, .release_cb = smc_release_cb, @@ -116,6 +120,18 @@ static int smc_inet_init_sock(struct sock *sk) return smc_create_clcsk(net, sk, sk->sk_family); } +static void smc_inet_destroy_sock(struct sock *sk) +{ + /* The sock is hashed and smc_diag dumps dereference smc->clcsock + * without clcsock_release_lock, while sk_common_release() calls + * .destroy before .unhash. Unhash first, as __smc_release() does, + * so no dump can observe the clcsock being released; the second + * unhash is a no-op. + */ + sk->sk_prot->unhash(sk); + smc_clcsock_release(smc_sk(sk)); +} + int __init smc_inet_init(void) { int rc; diff --git a/net/smc/smc_llc.c b/net/smc/smc_llc.c index 481b0495b0d315..c9ccd8480369ca 100644 --- a/net/smc/smc_llc.c +++ b/net/smc/smc_llc.c @@ -341,6 +341,7 @@ again: void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) { spin_lock_bh(&lgr->llc_flow_lock); + smc_llc_flow_qentry_del(flow); memset(flow, 0, sizeof(*flow)); flow->type = SMC_LLC_FLOW_NONE; spin_unlock_bh(&lgr->llc_flow_lock); @@ -1945,6 +1946,8 @@ static void smc_llc_event_handler(struct smc_llc_qentry *qentry) if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_REQ_ADD_LINK) { /* server started add_link processing */ + /* free any qentry stashed in REQ_ADD_LINK state */ + smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK; smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); diff --git a/net/sunrpc/rpcb_clnt.c b/net/sunrpc/rpcb_clnt.c index 6aa372188c86aa..4c0b7fefee4e2e 100644 --- a/net/sunrpc/rpcb_clnt.c +++ b/net/sunrpc/rpcb_clnt.c @@ -490,6 +490,8 @@ static int rpcb_register_inet4(struct sunrpc_net *sn, int result; map->r_addr = rpc_sockaddr2uaddr(sap, GFP_KERNEL); + if (!map->r_addr) + return -ENOMEM; msg->rpc_proc = &rpcb_procedures4[RPCBPROC_UNSET]; if (port != 0) { @@ -516,6 +518,8 @@ static int rpcb_register_inet6(struct sunrpc_net *sn, int result; map->r_addr = rpc_sockaddr2uaddr(sap, GFP_KERNEL); + if (!map->r_addr) + return -ENOMEM; msg->rpc_proc = &rpcb_procedures4[RPCBPROC_UNSET]; if (port != 0) { diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index 5f7955e4c4045f..7f60723fa64d84 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -1202,9 +1202,9 @@ static void xs_save_old_callbacks(struct sock_xprt *transport, struct sock *sk) static void xs_restore_old_callbacks(struct sock_xprt *transport, struct sock *sk) { - sk->sk_data_ready = transport->old_data_ready; + WRITE_ONCE(sk->sk_data_ready, transport->old_data_ready); sk->sk_state_change = transport->old_state_change; - sk->sk_write_space = transport->old_write_space; + WRITE_ONCE(sk->sk_write_space, transport->old_write_space); sk->sk_error_report = transport->old_error_report; } @@ -1664,6 +1664,7 @@ static void xs_udp_do_set_buffer_size(struct rpc_xprt *xprt) { struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); struct sock *sk = transport->inet; + void (*write_space)(struct sock *sock); if (transport->rcvsize) { sk->sk_userlocks |= SOCK_RCVBUF_LOCK; @@ -1672,7 +1673,8 @@ static void xs_udp_do_set_buffer_size(struct rpc_xprt *xprt) if (transport->sndsize) { sk->sk_userlocks |= SOCK_SNDBUF_LOCK; sk->sk_sndbuf = transport->sndsize * xprt->max_reqs * 2; - sk->sk_write_space(sk); + write_space = READ_ONCE(sk->sk_write_space); + write_space(sk); } } @@ -1988,8 +1990,8 @@ static int xs_local_finish_connecting(struct rpc_xprt *xprt, xs_save_old_callbacks(transport, sk); sk->sk_user_data = xprt; - sk->sk_data_ready = xs_data_ready; - sk->sk_write_space = xs_udp_write_space; + WRITE_ONCE(sk->sk_data_ready, xs_data_ready); + WRITE_ONCE(sk->sk_write_space, xs_udp_write_space); sk->sk_state_change = xs_local_state_change; sk->sk_error_report = xs_error_report; sk->sk_use_task_frag = false; @@ -2191,8 +2193,8 @@ static void xs_udp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock) xs_save_old_callbacks(transport, sk); sk->sk_user_data = xprt; - sk->sk_data_ready = xs_data_ready; - sk->sk_write_space = xs_udp_write_space; + WRITE_ONCE(sk->sk_data_ready, xs_data_ready); + WRITE_ONCE(sk->sk_write_space, xs_udp_write_space); sk->sk_use_task_frag = false; xprt_set_connected(xprt); @@ -2378,9 +2380,9 @@ static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock) xs_save_old_callbacks(transport, sk); sk->sk_user_data = xprt; - sk->sk_data_ready = xs_data_ready; + WRITE_ONCE(sk->sk_data_ready, xs_data_ready); sk->sk_state_change = xs_tcp_state_change; - sk->sk_write_space = xs_tcp_write_space; + WRITE_ONCE(sk->sk_write_space, xs_tcp_write_space); sk->sk_error_report = xs_error_report; sk->sk_use_task_frag = false; diff --git a/net/tls/tls_strp.c b/net/tls/tls_strp.c index 61b10c697eccbb..6cc222008d95c3 100644 --- a/net/tls/tls_strp.c +++ b/net/tls/tls_strp.c @@ -430,9 +430,10 @@ static int tls_strp_read_copy(struct tls_strparser *strp, bool qshort) return 0; } -static bool tls_strp_check_queue_ok(struct tls_strparser *strp) +static bool tls_strp_check_queue_ok(struct tls_strparser *strp, + unsigned int len) { - unsigned int len = strp->stm.offset + strp->stm.full_len; + unsigned int remaining = strp->stm.offset + len; struct sk_buff *first, *skb; u32 seq; @@ -443,9 +444,9 @@ static bool tls_strp_check_queue_ok(struct tls_strparser *strp) /* Make sure there's no duplicate data in the queue, * and the decrypted status matches. */ - while (skb->len < len) { + while (skb->len < remaining) { seq += skb->len; - len -= skb->len; + remaining -= skb->len; skb = skb->next; if (TCP_SKB_CB(skb)->seq != seq) @@ -525,6 +526,11 @@ static int tls_strp_read_sock(struct tls_strparser *strp) tls_strp_load_anchor_with_queue(strp, inq); if (!strp->stm.full_len) { + if (inq < TLS_HEADER_SIZE) + return tls_strp_read_copy(strp, true); + if (!tls_strp_check_queue_ok(strp, TLS_HEADER_SIZE)) + return tls_strp_read_copy(strp, false); + sz = tls_rx_msg_size(strp, strp->anchor); if (sz < 0) return sz; @@ -535,7 +541,7 @@ static int tls_strp_read_sock(struct tls_strparser *strp) return tls_strp_read_copy(strp, true); } - if (!tls_strp_check_queue_ok(strp)) + if (!tls_strp_check_queue_ok(strp, strp->stm.full_len)) return tls_strp_read_copy(strp, false); WRITE_ONCE(strp->msg_ready, 1); diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c index 622dbd04679944..78e60ad043633d 100644 --- a/net/vmw_vsock/af_vsock.c +++ b/net/vmw_vsock/af_vsock.c @@ -1847,12 +1847,10 @@ static int vsock_connect(struct socket *sock, struct sockaddr_unsized *addr, prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); } - if (sk->sk_err) { - err = -sk->sk_err; + err = sock_error(sk); + if (err) { sk->sk_state = TCP_CLOSE; sock->state = SS_UNCONNECTED; - } else { - err = 0; } out_wait: @@ -1893,7 +1891,7 @@ static int vsock_accept(struct socket *sock, struct socket *newsock, timeout = sock_rcvtimeo(listener, arg->flags & O_NONBLOCK); while ((connected = vsock_dequeue_accept(listener)) == NULL && - listener->sk_err == 0 && timeout != 0) { + timeout != 0) { prepare_to_wait(sk_sleep(listener), &wait, TASK_INTERRUPTIBLE); release_sock(listener); timeout = schedule_timeout(timeout); @@ -1906,13 +1904,9 @@ static int vsock_accept(struct socket *sock, struct socket *newsock, } } - if (listener->sk_err) { - err = -listener->sk_err; - } else if (!connected) { + if (!connected) { err = -EAGAIN; - } - - if (connected) { + } else { sk_acceptq_removed(listener); lock_sock_nested(connected, SINGLE_DEPTH_NESTING); diff --git a/net/wireless/core.c b/net/wireless/core.c index 610238d723fff7..d13310fef691ae 100644 --- a/net/wireless/core.c +++ b/net/wireless/core.c @@ -237,6 +237,7 @@ void cfg80211_stop_p2p_device(struct cfg80211_registered_device *rdev, if (!wdev_running(wdev)) return; + cfg80211_pmsr_wdev_down(wdev); rdev_stop_p2p_device(rdev, wdev); wdev->is_running = false; @@ -264,6 +265,8 @@ void cfg80211_stop_nan(struct cfg80211_registered_device *rdev, if (!wdev_running(wdev)) return; + cfg80211_pmsr_wdev_down(wdev); + /* * If there is a scheduled update pending, mark it as canceled, so the * empty schedule will be accepted diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 5adcb6bd0fc56d..755f8fe711fb2f 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -18750,15 +18750,15 @@ static int nl80211_color_change(struct sk_buff *skb, struct genl_info *info) if (!wdev->links[params.link_id].ap.beacon_interval) return -EINVAL; + tb = kzalloc_objs(*tb, NL80211_ATTR_MAX + 1); + if (!tb) + return -ENOMEM; + err = nl80211_parse_beacon(rdev, info->attrs, ¶ms.beacon_next, wdev->links[params.link_id].ap.chandef.chan, info->extack); if (err) - return err; - - tb = kzalloc_objs(*tb, NL80211_ATTR_MAX + 1); - if (!tb) - return -ENOMEM; + goto out; err = nla_parse_nested(tb, NL80211_ATTR_MAX, info->attrs[NL80211_ATTR_COLOR_CHANGE_ELEMS], diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 7855ee09c4b640..33475b180ea6f3 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -298,9 +298,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) u32 frame_size = __xsk_pool_get_rx_frame_size(xs->pool); void *copy_from = xsk_copy_xdp_start(xdp), *copy_to; u32 from_len, meta_len, rem, num_desc; - struct xdp_buff_xsk *xskb; + struct xdp_buff_xsk *xskb, *tmp; struct xdp_buff *xsk_xdp; + LIST_HEAD(xsk_buffs); skb_frag_t *frag; + u32 i; from_len = xdp->data_end - copy_from; meta_len = xdp->data - copy_from; @@ -343,23 +345,45 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) frag = &sinfo->frags[0]; } + for (i = 0; i < num_desc; i++) { + xsk_xdp = xsk_buff_alloc(xs->pool); + if (!xsk_xdp) + goto err_alloc; + + xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp); + if (unlikely(!list_empty(&xskb->list_node))) + goto err_alloc; + + list_add_tail(&xskb->list_node, &xsk_buffs); + } + do { u32 to_len = frame_size + meta_len; u32 copied; - xsk_xdp = xsk_buff_alloc(xs->pool); + xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk, + list_node); + list_del_init(&xskb->list_node); + xsk_xdp = &xskb->xdp; copy_to = xsk_xdp->data - meta_len; copied = xsk_copy_xdp(copy_to, ©_from, to_len, &from_len, &frag, rem); rem -= copied; - xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp); __xsk_rcv_zc_safe(xs, xskb, copied - meta_len, rem ? XDP_PKT_CONTD : 0); meta_len = 0; } while (rem); return 0; + +err_alloc: + list_for_each_entry_safe(xskb, tmp, &xsk_buffs, list_node) { + list_del_init(&xskb->list_node); + xsk_buff_free(&xskb->xdp); + } + xs->rx_dropped++; + return -ENOMEM; } static bool xsk_tx_writeable(struct xdp_sock *xs) diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c index 78c14f10639573..9d2d94f1fb7595 100644 --- a/net/xdp/xsk_buff_pool.c +++ b/net/xdp/xsk_buff_pool.c @@ -763,6 +763,7 @@ EXPORT_SYMBOL(xp_raw_get_dma); * xp_raw_get_ctx - get &xdp_desc context * @pool: XSk buff pool desc address belongs to * @addr: desc address (from userspace) + * @options: desc options (from userspace) * * Helper for getting desc's DMA address and metadata pointer, if present. * Saves one call on hotpath and double calculation of the actual address. @@ -771,14 +772,16 @@ EXPORT_SYMBOL(xp_raw_get_dma); * Return: new &xdp_desc_ctx struct containing desc's DMA address and metadata * pointer, if it is present (initialized to %NULL otherwise). */ -struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) +struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr, + u32 options) { struct xdp_desc_ctx ret; addr = __xp_raw_get_addr(pool, addr); ret.dma = __xp_raw_get_dma(pool, addr); - ret.meta = __xsk_buff_get_metadata(pool, __xp_raw_get_data(pool, addr)); + ret.meta = __xsk_buff_get_metadata(pool, __xp_raw_get_data(pool, addr), + options); return ret; } diff --git a/net/xfrm/xfrm_output.c b/net/xfrm/xfrm_output.c index cc35c2fcbbe095..e305ba32e356b7 100644 --- a/net/xfrm/xfrm_output.c +++ b/net/xfrm/xfrm_output.c @@ -636,10 +636,8 @@ static int xfrm_dev_direct_output(struct sock *sk, struct xfrm_state *x, nf_reset_ct(skb); err = skb_dst(skb)->ops->local_out(net, sk, skb); - if (unlikely(err != 1)) { - kfree_skb(skb); + if (unlikely(err != 1)) return err; - } /* In transport mode, network destination is * directly reachable, while in tunnel mode, diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 998e31052e6601..4b90a1390ad58c 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -58,7 +58,9 @@ #include "cred.c" #include "device.c" #include "dma.c" +#ifdef CONFIG_DMA_SHARED_BUFFER #include "dma-resv.c" +#endif #include "drm.c" #include "drm_gpuvm.c" #include "err.c" diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index 58ac04c650a11d..1158ef3a85f2fb 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -361,23 +361,28 @@ impl TableBuilder { } } - /// Adds a new entry to the table. - pub fn add(&mut self, freq: Hertz, flags: u32, driver_data: u32) -> Result { + /// Adds a raw frequency-table entry. + fn push(&mut self, frequency: u32, flags: u32, driver_data: u32) -> Result { // Adds the new entry at the end of the vector. Ok(self.entries.push( bindings::cpufreq_frequency_table { flags, driver_data, - frequency: freq.as_khz() as u32, + frequency, }, GFP_KERNEL, )?) } + /// Adds a new entry to the table. + pub fn add(&mut self, freq: Hertz, flags: u32, driver_data: u32) -> Result { + self.push(freq.as_khz() as u32, flags, driver_data) + } + /// Consumes the [`TableBuilder`] and returns [`TableBox`]. pub fn to_table(mut self) -> Result { // Add last entry to the table. - self.add(Hertz(c_ulong::MAX), 0, 0)?; + self.push(bindings::CPUFREQ_TABLE_END as u32, 0, 0)?; TableBox::new(self.entries) } @@ -817,7 +822,9 @@ pub trait Driver { } /// Driver's `bios_limit` callback. - fn bios_limit(_policy: &mut Policy, _limit: &mut u32) -> Result { + /// + /// Returns HW/BIOS max frequency limitations for the CPU. + fn bios_limit(_policy: &mut Policy) -> Result { build_error!(VTABLE_DEFAULT_ERROR) } @@ -1352,9 +1359,12 @@ impl Registration { from_result(|| { let mut policy = PolicyCpu::from_cpu(cpu_id)?; - + let val = T::bios_limit(&mut policy)?; // SAFETY: `limit` is guaranteed by the C code to be valid. - T::bios_limit(&mut policy, &mut (unsafe { *limit })).map(|()| 0) + unsafe { + *limit = val; + } + Ok(0) }) } diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 1a38b3bbdfb7d8..09ad8ec31548c2 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -511,7 +511,11 @@ pub struct Normal; /// callback it appears in. It is intended to be used for synchronization purposes. Bus device /// implementations can implement methods for [`Device`], such that they can only be called /// from bus callbacks. -pub struct Core<'a>(PhantomData<&'a ()>); +/// +/// The lifetime `'a` is for "lifetime branding" purpose. Callbacks need to polymorphic over this +/// lifetime so the `&'bound Device>` provided to them cannot outlive the scope of the +/// function. For this reason, it needs to be invariant. +pub struct Core<'a>(PhantomData &'a ()>); /// Semantically the same as [`Core`], but reserved for internal usage of the corresponding bus /// abstraction. @@ -522,7 +526,9 @@ pub struct Core<'a>(PhantomData<&'a ()>); /// /// This context mainly exists to share generic [`Device`] infrastructure that should only be called /// from bus callbacks with bus abstractions, but without making them accessible for drivers. -pub struct CoreInternal<'a>(PhantomData<&'a ()>); +/// +/// Lifetime `'a` is invariant for the same reason as [`Core`]. +pub struct CoreInternal<'a>(PhantomData &'a ()>); /// The [`Bound`] context is the [`DeviceContext`] of a bus specific device when it is guaranteed to /// be bound to a driver. diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs index ae58f6f667c1fa..70cf11346ceeec 100644 --- a/rust/kernel/drm/gpuvm/mod.rs +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -72,10 +72,12 @@ pub struct GpuVm { data: UnsafeCell, } -// SAFETY: The GPUVM api does not assume that it is tied to a specific thread. The destructor will -// drop the `data` field, which is okay because it is guaranteed `Send` by the `DriverGpuVm` trait. +// SAFETY: It is safe to send a `GpuVm` to another thread: all data reachable through it +// (`T`, `T::VmBoData`, and the GEM `T::Object`) is `Send` by the `DriverGpuVm` bounds. unsafe impl Send for GpuVm {} -// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel. +// SAFETY: It is safe to share a `&GpuVm` between threads: `&self` methods only alias data +// that is `Sync` by the `DriverGpuVm` bounds, and any thread may drop that data, or upgrade the +// reference and ultimately drop `T`, which the same bounds make `Send`. unsafe impl Sync for GpuVm {} // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. @@ -116,9 +118,9 @@ impl GpuVm { /// Creates a GPUVM instance. #[expect(clippy::new_ret_no_self)] - pub fn new( + pub fn new( name: &'static CStr, - dev: &drm::Device, + dev: &drm::Device, r_obj: &T::Object, range: Range, reserve_range: Range, @@ -250,18 +252,22 @@ impl GpuVm { } /// The manager for a GPUVM. -pub trait DriverGpuVm: Sized + Send { +pub trait DriverGpuVm: Sized + Send + Sync { /// Parent `Driver` for this object. - type Driver: drm::Driver; + type Driver: drm::Driver; /// The kind of GEM object stored in this GPUVM. - type Object: IntoGEMObject; + type Object: drm::driver::AllocImpl + Send + Sync; /// Data stored with each [`struct drm_gpuva`](struct@GpuVa). - type VaData; + /// + /// Only `Send` is required: the data has a single owner at all times, moving + /// between threads by value (handed back as a [`GpuVaRemoved`]) but never + /// accessed by two threads concurrently. + type VaData: Send; /// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo). - type VmBoData; + type VmBoData: Send + Sync; /// The private data passed to callbacks. type SmContext<'ctx>; @@ -296,12 +302,10 @@ pub trait DriverGpuVm: Sized + Send { /// # Invariants /// /// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference. +// `Send`/`Sync` derive from `ARef>`; the trait bounds make them correct for the unique +// handle's `&mut T` access. pub struct UniqueRefGpuVm(ARef>); -// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel, and -// concurrent access to `data` is safe due to the `T: Sync` requirement. -unsafe impl Sync for UniqueRefGpuVm {} - impl UniqueRefGpuVm { /// Access the data owned by this `UniqueRefGpuVm` immutably. #[inline] diff --git a/rust/uapi/uapi_helper.h b/rust/uapi/uapi_helper.h index 06d7d1a2e8daba..1c4aa4292dcec9 100644 --- a/rust/uapi/uapi_helper.h +++ b/rust/uapi/uapi_helper.h @@ -6,11 +6,11 @@ * Sorted alphabetically. */ -#include #include #include #include #include +#include #include #include #include diff --git a/samples/ftrace/ftrace-direct-modify.c b/samples/ftrace/ftrace-direct-modify.c index 1ba1927b548ee9..164d9dd6fd921d 100644 --- a/samples/ftrace/ftrace-direct-modify.c +++ b/samples/ftrace/ftrace-direct-modify.c @@ -320,9 +320,15 @@ static int __init ftrace_direct_init(void) ftrace_set_filter_ip(&direct, (unsigned long) my_ip, 0, 0); ret = register_ftrace_direct(&direct, my_tramp); - if (!ret) - simple_tsk = kthread_run(simple_thread, NULL, "event-sample-fn"); - return ret; + if (ret) + return ret; + simple_tsk = kthread_run(simple_thread, NULL, "event-sample-fn"); + if (IS_ERR(simple_tsk)) { + unregister_ftrace_direct(&direct, my_tramp, true); + return PTR_ERR(simple_tsk); + } + + return 0; } static void __exit ftrace_direct_exit(void) diff --git a/samples/ftrace/ftrace-direct-multi-modify.c b/samples/ftrace/ftrace-direct-multi-modify.c index 7a7822dfeb50ab..b03766c6217bcc 100644 --- a/samples/ftrace/ftrace-direct-multi-modify.c +++ b/samples/ftrace/ftrace-direct-multi-modify.c @@ -364,9 +364,15 @@ static int __init ftrace_direct_multi_init(void) ret = register_ftrace_direct(&direct, my_tramp); - if (!ret) - simple_tsk = kthread_run(simple_thread, NULL, "event-sample-fn"); - return ret; + if (ret) + return ret; + simple_tsk = kthread_run(simple_thread, NULL, "event-sample-fn"); + if (IS_ERR(simple_tsk)) { + unregister_ftrace_direct(&direct, my_tramp, true); + return PTR_ERR(simple_tsk); + } + + return 0; } static void __exit ftrace_direct_multi_exit(void) diff --git a/samples/rust/rust_debugfs.rs b/samples/rust/rust_debugfs.rs index 1f59e08aaa4b0d..0b27ad96ecbfaf 100644 --- a/samples/rust/rust_debugfs.rs +++ b/samples/rust/rust_debugfs.rs @@ -147,7 +147,9 @@ impl RustDebugFs { dir.read_write_file(c"pair", new_mutex!(Inner { x: 3, y: 10 })) } - fn new<'a>(pdev: &'a platform::Device>) -> impl PinInit + 'a { + fn new<'a, 'b>( + pdev: &'a platform::Device>, + ) -> impl PinInit + use<'a, 'b> { let debugfs = Dir::new(c"sample_debugfs"); let dev = pdev.as_ref(); diff --git a/samples/rust/rust_debugfs_scoped.rs b/samples/rust/rust_debugfs_scoped.rs index 6a575a15a2c2f9..ca2b154be3842a 100644 --- a/samples/rust/rust_debugfs_scoped.rs +++ b/samples/rust/rust_debugfs_scoped.rs @@ -75,7 +75,10 @@ fn create_file_write( GFP_KERNEL, )?; } - let blob = KBox::pin_init(new_mutex!([0x42; SZ_4K]), GFP_KERNEL)?; + let blob = KBox::pin_init( + new_mutex!(pin_init::init_array_from_fn(|_| 0x42)), + GFP_KERNEL, + )?; let scope = KBox::pin_init( mod_data.device_dir.scope( diff --git a/scripts/Makefile.vmlinux_o b/scripts/Makefile.vmlinux_o index 527352c222ff63..24a3a4fd271c23 100644 --- a/scripts/Makefile.vmlinux_o +++ b/scripts/Makefile.vmlinux_o @@ -47,6 +47,9 @@ endif vmlinux-objtool-args-$(CONFIG_NOINSTR_VALIDATION) += --noinstr \ $(if $(or $(CONFIG_MITIGATION_UNRET_ENTRY),$(CONFIG_MITIGATION_SRSO)), --unret) +# Only used for builds initiated by klp-build +vmlinux-objtool-args-$(if $(KLP_SYMIDS),y) += --klp-symids + objtool-args = $(vmlinux-objtool-args-y) --link # Link of vmlinux.o used for section mismatch analysis diff --git a/scripts/livepatch/klp-build b/scripts/livepatch/klp-build index c4a7acf8edc3f8..30cd881ca7ed1e 100755 --- a/scripts/livepatch/klp-build +++ b/scripts/livepatch/klp-build @@ -271,6 +271,9 @@ validate_config() { [[ -v CONFIG_GCC_PLUGIN_RANDSTRUCT ]] && \ die "kernel option 'CONFIG_GCC_PLUGIN_RANDSTRUCT' not supported" + [[ -v CONFIG_LD_DEAD_CODE_DATA_ELIMINATION ]] && \ + die "kernel option 'CONFIG_LD_DEAD_CODE_DATA_ELIMINATION' not supported" + [[ -v CONFIG_AS_IS_LLVM ]] && \ [[ "$CONFIG_AS_VERSION" -lt 200000 ]] && \ die "Clang assembler version < 20 not supported" @@ -555,6 +558,8 @@ build_kernel() { # cmd+=("KBUILD_MODPOST_WARN=1") + cmd+=("KLP_SYMIDS=1") + if [[ -v VERBOSE ]]; then cmd+=("V=1") else @@ -605,6 +610,8 @@ copy_orig_objects() { done xtrace_restore + cp -f "$PWD/vmlinux" "$ORIG_DIR" || die "missing vmlinux" + mv -f "$TMP_DIR/build.log" "$ORIG_DIR" touch "$TIMESTAMP" touch "$ORIG_DIR/.complete" @@ -675,6 +682,8 @@ generate_checksums() { "$OBJTOOL" klp checksum "$dest" done + [[ -f "$src_dir/vmlinux" ]] && cp -f "$src_dir/vmlinux" "$dest_dir" + touch "$dest_dir/.complete" } diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index a7b72a81d2482a..91b35eec69a8b2 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -767,6 +767,7 @@ static const char *const section_white_list[] = ".llvm.call-graph-profile", /* call graph */ "__llvm_covfun", "__llvm_covmap", + ".klp.symid", /* objtool --klp-symids */ NULL }; @@ -1591,14 +1592,14 @@ static void read_symbols(const char *modname) struct elf_info info = { }; Elf_Sym *sym; - if (!parse_elf(&info, modname)) - return; - if (!strends(modname, ".o")) { error("%s: filename must be suffixed with .o\n", modname); return; } + if (!parse_elf(&info, modname)) + return; + /* strip trailing .o */ mod = new_module(modname, strlen(modname) - strlen(".o")); diff --git a/scripts/tags.sh b/scripts/tags.sh index 243373683f98a1..c9dc2763a505e6 100755 --- a/scripts/tags.sh +++ b/scripts/tags.sh @@ -100,7 +100,7 @@ all_compiled_sources() { echo include/generated/autoconf.h find $ignore -name "*.cmd" -exec \ - grep -Poh '(?<=^ )\S+|(?<== )\S+[^\\](?=$)' {} \+ | + grep -Poh '(?<=^ )\S+\.([chS]|rs)(?=\s)|(?<== )\S+\.(?1)(?=$)' {} \+ | awk '!a[$0]++' } | xargs realpath -esq $([ -z "$KBUILD_ABS_SRCTREE" ] && echo --relative-to=.) | sort -u diff --git a/security/apparmor/domain.c b/security/apparmor/domain.c index d6958eb00e3045..e88c19c9b175a9 100644 --- a/security/apparmor/domain.c +++ b/security/apparmor/domain.c @@ -874,6 +874,52 @@ static struct aa_label *label_merge_wrap(struct aa_label *a, struct aa_label *b, return label; } +static bool is_profile_priv_restricted_to_stack(const struct cred *subj_cred, + struct aa_profile *profile) +{ + if (profile_unconfined(profile) && profile == profile->ns->unconfined && + aa_unprivileged_unconfined_restricted && + /* cap_capable returns false (0) if true, hence true here means + * doesn't have capability and the stack will be restricted + */ + cap_capable(current_cred(), &init_user_ns, CAP_MAC_OVERRIDE, + CAP_OPT_NOAUDIT)) + return true; + return false; +} + +static const char *stack_msg = "change_profile unprivileged unconfined converted to stacking"; + +static struct aa_label *priv_restricted_transition(const struct cred *subj_cred, + struct aa_profile *profile, + const char *op, u32 request, + const char *name, + struct aa_label *transition, + gfp_t gfp) +{ + if (!is_profile_priv_restricted_to_stack(subj_cred, profile)) + return aa_get_newest_label(transition); + + /* transition allowed but only via stack */ + struct aa_label *target = label_merge_wrap(&profile->label, + transition, gfp); + if (IS_ERR_OR_NULL(target)) + return target; + + /* doing this here is less than optimal but good enough until the + * fs mediation rework lands + */ + struct aa_perms perms = { + .allow = request, + .audit = request, + }; + aa_audit_file(subj_cred, profile, &perms, op, + request, name, NULL, target, + subj_cred->euid, stack_msg, 0); + + return target; +} + static struct aa_label *handle_onexec(const struct cred *subj_cred, struct aa_label *label, struct aa_label *onexec, bool stack, @@ -902,7 +948,10 @@ static struct aa_label *handle_onexec(const struct cred *subj_cred, new = fn_label_build_in_scope(label, profile, GFP_KERNEL, stack ? label_merge_wrap(&profile->label, onexec, GFP_KERNEL) - : aa_get_newest_label(onexec), + : priv_restricted_transition(subj_cred, profile, + OP_CHANGE_ONEXEC, AA_MAY_ONEXEC, + bprm->filename, onexec, + GFP_KERNEL), profile_transition(subj_cred, profile, bprm, buffer, cond, unsafe)); AA_BUG(!new); @@ -1084,8 +1133,8 @@ static struct aa_label *build_change_hat(const struct cred *subj_cred, if (!hat) { error = -ENOENT; if (COMPLAIN_MODE(profile)) { - hat = aa_new_learning_profile(profile, true, name, - GFP_KERNEL); + hat = __aa_new_learning_profile(profile, true, name, + GFP_KERNEL); if (!hat) { info = "failed null profile create"; error = -ENOMEM; @@ -1123,6 +1172,7 @@ static struct aa_label *change_hat(const struct cred *subj_cred, bool sibling = false; const char *name, *info = NULL; int i, error; + bool needput = false; AA_BUG(!label); AA_BUG(!hats); @@ -1135,7 +1185,6 @@ static struct aa_label *change_hat(const struct cred *subj_cred, * the profiles and label, we can rely on the namespaces being live * and avoid incrementing their refcounts while grabbing the lock. */ - label = aa_get_label(label); ns = labels_ns(label); retry: @@ -1143,15 +1192,19 @@ retry: if (label_is_stale(label)) { new = aa_get_newest_label(label); new_ns = labels_ns(new); + + if (needput) + /* aa_put_label() is safe to call when under lock */ + aa_put_label(label); + label = new; + needput = true; + /* check if replaced with label in parent ns, and lock there */ if (new_ns != ns) { - aa_put_label(new); mutex_unlock(&ns->lock); ns = new_ns; - label = new; + /* retry will bottom out at the root of the tree */ goto retry; } - aa_put_label(label); - label = new; } if (PROFILE_IS_HAT(labels_profile(label))) @@ -1162,7 +1215,8 @@ retry: name = hats[i]; label_for_each_in_scope(it, labels_ns(label), label, profile) { if (sibling && PROFILE_IS_HAT(profile)) { - root = aa_get_profile(profile->parent); + root = aa_get_profile(rcu_dereference_protected(profile->parent, + mutex_is_locked(&ns->lock))); } else if (!sibling && !PROFILE_IS_HAT(profile)) { root = aa_get_profile(profile); } else { /* conflicting change type */ @@ -1223,6 +1277,8 @@ fail: } } mutex_unlock(&ns->lock); + if (needput) + aa_put_label(label); return ERR_PTR(error); build: @@ -1233,7 +1289,8 @@ build: mutex_unlock(&ns->lock); AA_BUG(!new); /* return new label or error ptr */ - + if (needput) + aa_put_label(label); return new; } @@ -1406,8 +1463,6 @@ static int change_profile_perms_wrapper(const char *op, const char *name, return error; } -static const char *stack_msg = "change_profile unprivileged unconfined converted to stacking"; - /** * aa_change_profile - perform a one-way profile transition * @fqname: name of profile may include namespace (NOT NULL) @@ -1467,28 +1522,6 @@ int aa_change_profile(const char *fqname, int flags) op = OP_CHANGE_PROFILE; } - /* This should move to a per profile test. Requires pushing build - * into callback - */ - if (!stack && unconfined(label) && - label == &labels_ns(label)->unconfined->label && - aa_unprivileged_unconfined_restricted && - /* TODO: refactor so this check is a fn */ - cap_capable(current_cred(), &init_user_ns, CAP_MAC_OVERRIDE, - CAP_OPT_NOAUDIT)) { - /* regardless of the request in this case apparmor - * stacks against unconfined so admin set policy can't be - * by-passed - */ - stack = true; - perms.audit = request; - (void) fn_for_each_in_scope(label, profile, - aa_audit_file(subj_cred, profile, &perms, op, - request, auditname, NULL, target, - GLOBAL_ROOT_UID, stack_msg, 0)); - perms.audit = 0; - } - if (*fqname == '&') { stack = true; /* don't have label_parse() do stacking */ @@ -1559,7 +1592,10 @@ check: /* stacking is always a subset, so only check the nonstack case */ if (!stack) { new = fn_label_build_in_scope(label, profile, GFP_KERNEL, - aa_get_label(target), + priv_restricted_transition(subj_cred, profile, + op, request, + auditname, target, + GFP_KERNEL), aa_get_label(&profile->label)); AA_BUG(!new); if (IS_ERR(new)) diff --git a/security/apparmor/include/policy.h b/security/apparmor/include/policy.h index 3895f8774a3f93..c6439ce6e436a1 100644 --- a/security/apparmor/include/policy.h +++ b/security/apparmor/include/policy.h @@ -295,6 +295,9 @@ struct aa_profile *aa_alloc_profile(const char *name, struct aa_proxy *proxy, gfp_t gfp); struct aa_profile *aa_alloc_null(struct aa_profile *parent, const char *name, gfp_t gfp); +struct aa_profile *__aa_new_learning_profile(struct aa_profile *parent, + bool hat, const char *base, + gfp_t gfp); struct aa_profile *aa_new_learning_profile(struct aa_profile *parent, bool hat, const char *base, gfp_t gfp); void aa_free_profile(struct aa_profile *profile); diff --git a/security/apparmor/lib.c b/security/apparmor/lib.c index e41ff57798b25e..b284cfcc5d3497 100644 --- a/security/apparmor/lib.c +++ b/security/apparmor/lib.c @@ -482,6 +482,8 @@ bool aa_policy_init(struct aa_policy *policy, const char *prefix, char *hname; size_t hname_sz; + INIT_LIST_HEAD(&policy->list); + INIT_LIST_HEAD(&policy->profiles); hname_sz = (prefix ? strlen(prefix) + 2 : 0) + strlen(name) + 1; /* freed by policy_free */ hname = aa_str_alloc(hname_sz, gfp); @@ -494,8 +496,6 @@ bool aa_policy_init(struct aa_policy *policy, const char *prefix, policy->hname = hname; /* base.name is a substring of fqname */ policy->name = basename(policy->hname); - INIT_LIST_HEAD(&policy->list); - INIT_LIST_HEAD(&policy->profiles); return true; } diff --git a/security/apparmor/policy.c b/security/apparmor/policy.c index 94b4a7e727cc1d..cd87370578dda6 100644 --- a/security/apparmor/policy.c +++ b/security/apparmor/policy.c @@ -719,7 +719,7 @@ struct aa_profile *aa_alloc_null(struct aa_profile *parent, const char *name, } /** - * aa_new_learning_profile - create or find a null-X learning profile + * __aa_new_learning_profile - create or find a null-X learning profile * @parent: profile that caused this profile to be created (NOT NULL) * @hat: true if the null- learning profile is a hat * @base: name to base the null profile off of @@ -736,8 +736,9 @@ struct aa_profile *aa_alloc_null(struct aa_profile *parent, const char *name, * * Returns: new refcounted profile else NULL on failure */ -struct aa_profile *aa_new_learning_profile(struct aa_profile *parent, bool hat, - const char *base, gfp_t gfp) +struct aa_profile *__aa_new_learning_profile(struct aa_profile *parent, + bool hat, const char *base, + gfp_t gfp) { struct aa_profile *p, *profile; const char *bname; @@ -745,6 +746,7 @@ struct aa_profile *aa_new_learning_profile(struct aa_profile *parent, bool hat, size_t name_sz; AA_BUG(!parent); + AA_BUG(!mutex_is_locked(&parent->ns->lock)); if (base) { name_sz = strlen(parent->base.hname) + 8 + strlen(base); @@ -778,7 +780,6 @@ name: if (hat) profile->label.flags |= FLAG_HAT; - mutex_lock_nested(&profile->ns->lock, profile->ns->level); p = __find_child(&parent->base.profiles, bname); if (p) { aa_free_profile(profile); @@ -786,7 +787,6 @@ name: } else { __add_profile(&parent->base.profiles, profile); } - mutex_unlock(&profile->ns->lock); /* refcount released by caller */ out: @@ -800,6 +800,18 @@ fail: return NULL; } +struct aa_profile *aa_new_learning_profile(struct aa_profile *parent, bool hat, + const char *base, gfp_t gfp) +{ + struct aa_profile *profile; + + mutex_lock_nested(&parent->ns->lock, parent->ns->level); + profile = __aa_new_learning_profile(parent, hat, base, gfp); + mutex_unlock(&parent->ns->lock); + + return profile; +} + /** * replacement_allowed - test to see if replacement is allowed * @profile: profile to test if it can be replaced (MAYBE NULL) diff --git a/security/apparmor/policy_unpack.c b/security/apparmor/policy_unpack.c index d9dcff167c4801..15b5e9270c0faf 100644 --- a/security/apparmor/policy_unpack.c +++ b/security/apparmor/policy_unpack.c @@ -730,7 +730,7 @@ static bool verify_tags(struct aa_tags_struct *tags, const char **info) /* count followed by count indexes into hdrs */ u32 cnt = tags->sets.table[i]; - if (i+cnt >= tags->sets.size) { + if ((u64)i + cnt >= tags->sets.size) { AA_DEBUG(DEBUG_UNPACK, "tagset too large %d+%d > sets.table[%d]", i, cnt, tags->sets.size); diff --git a/security/smack/smack.h b/security/smack/smack.h index 9b9eb262fe33e6..fe6a498200143a 100644 --- a/security/smack/smack.h +++ b/security/smack/smack.h @@ -317,8 +317,9 @@ int smack_populate_secattr(struct smack_known *skp); * Shared data. */ extern int smack_enabled __initdata; -extern int smack_cipso_direct; -extern int smack_cipso_mapped; +extern u8 smack_cipso_auto_level[2]; +#define smack_cipso_direct (+smack_cipso_auto_level[0]) +#define smack_cipso_mapped (+smack_cipso_auto_level[1]) extern struct smack_known *smack_net_ambient; extern struct smack_known *smack_syslog_label; #ifdef CONFIG_SECURITY_SMACK_BRINGUP diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index bbe6cd6b03f75a..9a706f37df36b5 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -130,12 +130,13 @@ static int smk_bu_note(char *note, struct smack_known *sskp, #define smk_bu_note(note, sskp, oskp, mode, RC) (RC) #endif -#ifdef CONFIG_SECURITY_SMACK_BRINGUP -static int smk_bu_current(char *note, struct smack_known *oskp, - int mode, int rc) +static int +smk_bu_tsk_to_obj(struct task_struct *tsk, const struct task_smack *tsp, + char *note, struct smack_known *oskp, int mode, int rc) { - struct task_smack *tsp = smack_cred(current_cred()); +#ifdef CONFIG_SECURITY_SMACK_BRINGUP char acc[SMK_NUM_ACCESS_TYPE + 1]; + char comm[TASK_COMM_LEN]; if (rc <= 0) return rc; @@ -143,14 +144,22 @@ static int smk_bu_current(char *note, struct smack_known *oskp, rc = 0; smk_bu_mode(mode, acc); + pr_info("Smack %s: (%s %s %s) %s %s\n", smk_bu_mess[rc], - tsp->smk_task->smk_known, oskp->smk_known, - acc, current->comm, note); + smk_of_task(tsp)->smk_known, oskp->smk_known, + acc, get_task_comm(comm, tsk), note); return 0; -} #else -#define smk_bu_current(note, oskp, mode, RC) (RC) + return rc; #endif +} + +static int smk_bu_current(char *note, struct smack_known *oskp, + int mode, int rc) +{ + return smk_bu_tsk_to_obj(current, smack_cred(current_cred()), + note, oskp, mode, rc); +} #ifdef CONFIG_SECURITY_SMACK_BRINGUP static int smk_bu_task(struct task_struct *otp, int mode, int rc) @@ -1312,7 +1321,7 @@ static int smack_inode_getattr(const struct path *path) */ static int smack_inode_xattr_skipcap(const char *name) { - if (strncmp(name, XATTR_SMACK_SUFFIX, strlen(XATTR_SMACK_SUFFIX))) + if (strncmp(name, XATTR_SMACK_SUFFIX, strlen(XATTR_SMACK_SUFFIX)) == 0) return 0; if (strcmp(name, XATTR_NAME_SMACK) == 0 || @@ -3348,14 +3357,20 @@ static int smack_sem_semop(struct kern_ipc_perm *isp, struct sembuf *sops, } /** - * smk_curacc_msq : helper to check if current has access on msq - * @isp : the msq + * smk_tskacc_msq : helper to check if tsk has access on msq + * @tsk: the task that requests access + * @isp : the sysv msg queue permissions * @access : access requested * - * return 0 if current has access, error otherwise + * return 0 if tsk has access, error otherwise */ -static int smk_curacc_msq(struct kern_ipc_perm *isp, int access) +static int +smk_tskacc_msq(struct task_struct *tsk, struct kern_ipc_perm *isp, int access) { + const bool tsk_is_current = (tsk == current); + const struct cred * const tsk_cred = + (tsk_is_current ? current_cred() : get_task_cred(tsk)); + struct task_smack * const tsp = smack_cred(tsk_cred); struct smack_known *msp = smack_of_ipc(isp); struct smk_audit_info ad; int rc; @@ -3364,11 +3379,25 @@ static int smk_curacc_msq(struct kern_ipc_perm *isp, int access) smk_ad_init(&ad, __func__, LSM_AUDIT_DATA_IPC); ad.a.u.ipc_id = isp->id; #endif - rc = smk_curacc(msp, access, &ad); - rc = smk_bu_current("msq", msp, access, rc); + rc = smk_tskacc(tsp, msp, access, &ad); + rc = smk_bu_tsk_to_obj(tsk, tsp, "msq", msp, access, rc); + if (!tsk_is_current) + put_cred(tsk_cred); return rc; } +/** + * smk_curacc_msq : helper to check if current has access on msq + * @isp : the sysv msg queue permissions + * @access : access requested + * + * return 0 if current has access, error otherwise + */ +static int smk_curacc_msq(struct kern_ipc_perm *isp, int access) +{ + return smk_tskacc_msq(current, isp, access); +} + /** * smack_msg_queue_associate - Smack access check for msg_queue * @isp: the object @@ -3436,21 +3465,21 @@ static int smack_msg_queue_msgsnd(struct kern_ipc_perm *isp, struct msg_msg *msg } /** - * smack_msg_queue_msgrcv - Smack access check for msg_queue + * smack_msg_queue_msgrcv - check it target has r/w access to msg_queue * @isp: the object * @msg: unused - * @target: unused + * @target: the task that msgrcv() from the queue * @type: unused * @mode: unused * - * Returns 0 if current has read and write access, error code otherwise + * Returns 0 if target has read and write access, error code otherwise */ static int smack_msg_queue_msgrcv(struct kern_ipc_perm *isp, struct msg_msg *msg, struct task_struct *target, long type, int mode) { - return smk_curacc_msq(isp, MAY_READWRITE); + return smk_tskacc_msq(target, isp, MAY_READWRITE); } /** diff --git a/security/smack/smackfs.c b/security/smack/smackfs.c index 6e62dcb36f74f4..c7eae7c6427ff7 100644 --- a/security/smack/smackfs.c +++ b/security/smack/smackfs.c @@ -83,18 +83,27 @@ static DEFINE_MUTEX(smk_net6addr_lock); struct smack_known *smack_net_ambient; /* - * This is the level in a CIPSO header that indicates a + * Sensitivity levels for automatically created CIPSO labels. + * See smack_access.c`smack_populate_secattr() + * + * [0] "direct" labeling, label length < SMK_CIPSOLEN(24): * smack label is contained directly in the category set. * It can be reset via smackfs/direct - */ -int smack_cipso_direct = SMACK_CIPSO_DIRECT_DEFAULT; - -/* - * This is the level in a CIPSO header that indicates a + * + * [1] "mapped" labeling, label length >= SMK_CIPSOLEN(24): * secid is contained directly in the category set. * It can be reset via smackfs/mapped */ -int smack_cipso_mapped = SMACK_CIPSO_MAPPED_DEFAULT; +u8 smack_cipso_auto_level[2] = { + SMACK_CIPSO_DIRECT_DEFAULT, + SMACK_CIPSO_MAPPED_DEFAULT, +}; + +static int +smk_cipso_auto_level_idx(const struct file *file) +{ + return (file_inode(file)->i_ino != SMK_DIRECT); +} #ifdef CONFIG_SECURITY_SMACK_BRINGUP /* @@ -1598,24 +1607,17 @@ static ssize_t smk_read_doi(struct file *filp, char __user *buf, static ssize_t smk_write_doi(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - char temp[80]; - unsigned long u; + int ret; + u32 u; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; - if (count >= sizeof(temp) || count == 0) - return -EINVAL; - - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; + ret = kstrtou32_from_user(buf, count, 10, &u); + if (unlikely(ret)) + return ret; - temp[count] = '\0'; - - if (kstrtoul(temp, 10, &u)) - return -EINVAL; - - if (u == CIPSO_V4_DOI_UNKNOWN || u > U32_MAX) + if (u == CIPSO_V4_DOI_UNKNOWN) return -EINVAL; return smk_cipso_doi(u, GFP_KERNEL) ? : count; @@ -1628,158 +1630,80 @@ static const struct file_operations smk_doi_ops = { }; /** - * smk_read_direct - read() for /smack/direct - * @filp: file pointer, not actually used + * smk_read_cipso_auto_level - read() for smackfs/direct and smackfs/mapped + * @filp: file pointer * @buf: where to put the result * @count: maximum to send along * @ppos: where to start * * Returns number of bytes read or error code, as appropriate */ -static ssize_t smk_read_direct(struct file *filp, char __user *buf, +static ssize_t smk_read_cipso_auto_level(struct file *filp, char __user *buf, size_t count, loff_t *ppos) { - char temp[80]; - ssize_t rc; + char temp[sizeof "255"]; + int n; if (*ppos != 0) return 0; - sprintf(temp, "%d", smack_cipso_direct); - rc = simple_read_from_buffer(buf, count, ppos, temp, strlen(temp)); - - return rc; + n = sprintf(temp, "%u", (unsigned int)smack_cipso_auto_level[ + smk_cipso_auto_level_idx(filp)]); + return simple_read_from_buffer(buf, count, ppos, temp, n); } /** - * smk_write_direct - write() for /smack/direct - * @file: file pointer, not actually used + * smk_write_cipso_auto_level - write() for smackfs/direct and smackfs/mapped + * @filp: file pointer * @buf: where to get the data from * @count: bytes sent * @ppos: where to start * * Returns number of bytes written or error code, as appropriate */ -static ssize_t smk_write_direct(struct file *file, const char __user *buf, - size_t count, loff_t *ppos) +static ssize_t +smk_write_cipso_auto_level(struct file *filp, const char __user *buf, + size_t count, loff_t *ppos) { - struct smack_known *skp; - char temp[80]; - int i; + int ret, idx; + u8 i, old_lvl; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; - - if (count >= sizeof(temp) || count == 0) - return -EINVAL; - - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; - - temp[count] = '\0'; - - if (sscanf(temp, "%d", &i) != 1) - return -EINVAL; - /* - * Don't do anything if the value hasn't actually changed. - * If it is changing reset the level on entries that were - * set up to be direct when they were created. + * draft-ietf-cipso-ipsecurity-01 (CIPSO 2.2), 3.4.2.4: + * "Sensitivity Level is 1 octet in length. Its value is from 0 to 255" */ - if (smack_cipso_direct != i) { - mutex_lock(&smack_known_lock); - list_for_each_entry_rcu(skp, &smack_known_list, list) - if (skp->smk_netlabel.attr.mls.lvl == - smack_cipso_direct) - skp->smk_netlabel.attr.mls.lvl = i; - smack_cipso_direct = i; - mutex_unlock(&smack_known_lock); - } - - return count; -} - -static const struct file_operations smk_direct_ops = { - .read = smk_read_direct, - .write = smk_write_direct, - .llseek = default_llseek, -}; - -/** - * smk_read_mapped - read() for /smack/mapped - * @filp: file pointer, not actually used - * @buf: where to put the result - * @count: maximum to send along - * @ppos: where to start - * - * Returns number of bytes read or error code, as appropriate - */ -static ssize_t smk_read_mapped(struct file *filp, char __user *buf, - size_t count, loff_t *ppos) -{ - char temp[80]; - ssize_t rc; - - if (*ppos != 0) - return 0; - - sprintf(temp, "%d", smack_cipso_mapped); - rc = simple_read_from_buffer(buf, count, ppos, temp, strlen(temp)); - - return rc; -} - -/** - * smk_write_mapped - write() for /smack/mapped - * @file: file pointer, not actually used - * @buf: where to get the data from - * @count: bytes sent - * @ppos: where to start - * - * Returns number of bytes written or error code, as appropriate - */ -static ssize_t smk_write_mapped(struct file *file, const char __user *buf, - size_t count, loff_t *ppos) -{ - struct smack_known *skp; - char temp[80]; - int i; - - if (!smack_privileged(CAP_MAC_ADMIN)) - return -EPERM; - - if (count >= sizeof(temp) || count == 0) - return -EINVAL; - - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; - - temp[count] = '\0'; - - if (sscanf(temp, "%d", &i) != 1) - return -EINVAL; + ret = kstrtou8_from_user(buf, count, 10, &i); + if (unlikely(ret)) + return ret; /* * Don't do anything if the value hasn't actually changed. * If it is changing reset the level on entries that were - * set up to be mapped when they were created. + * set up to be "auto" level when they were created. */ - if (smack_cipso_mapped != i) { + idx = smk_cipso_auto_level_idx(filp); + old_lvl = smack_cipso_auto_level[idx]; + + if (old_lvl != i) { + struct smack_known *skp; mutex_lock(&smack_known_lock); list_for_each_entry_rcu(skp, &smack_known_list, list) if (skp->smk_netlabel.attr.mls.lvl == - smack_cipso_mapped) + old_lvl) skp->smk_netlabel.attr.mls.lvl = i; - smack_cipso_mapped = i; + smack_cipso_auto_level[idx] = i; mutex_unlock(&smack_known_lock); } return count; } -static const struct file_operations smk_mapped_ops = { - .read = smk_read_mapped, - .write = smk_write_mapped, +static const struct file_operations +smk_cipso_auto_level_ops = { + .read = smk_read_cipso_auto_level, + .write = smk_write_cipso_auto_level, .llseek = default_llseek, }; @@ -2179,22 +2103,15 @@ static ssize_t smk_read_logging(struct file *filp, char __user *buf, static ssize_t smk_write_logging(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - char temp[32]; - int i; + int i, ret; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; - if (count >= sizeof(temp) || count == 0) - return -EINVAL; + ret = kstrtos32_from_user(buf, count, 10, &i); + if (unlikely(ret)) + return ret; - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; - - temp[count] = '\0'; - - if (sscanf(temp, "%d", &i) != 1) - return -EINVAL; if (i < 0 || i > 3) return -EINVAL; log_policy = i; @@ -2838,22 +2755,15 @@ static ssize_t smk_read_ptrace(struct file *filp, char __user *buf, static ssize_t smk_write_ptrace(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - char temp[32]; - int i; + int i, ret; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; - if (*ppos != 0 || count >= sizeof(temp) || count == 0) - return -EINVAL; - - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; - - temp[count] = '\0'; + ret = kstrtos32_from_user(buf, count, 10, &i); + if (unlikely(ret)) + return ret; - if (sscanf(temp, "%d", &i) != 1) - return -EINVAL; if (i < SMACK_PTRACE_DEFAULT || i > SMACK_PTRACE_MAX) return -EINVAL; smack_ptrace_rule = i; @@ -2888,7 +2798,7 @@ static int smk_fill_super(struct super_block *sb, struct fs_context *fc) [SMK_DOI] = { "doi", &smk_doi_ops, S_IRUGO|S_IWUSR}, [SMK_DIRECT] = { - "direct", &smk_direct_ops, S_IRUGO|S_IWUSR}, + "direct", &smk_cipso_auto_level_ops, 0644}, [SMK_AMBIENT] = { "ambient", &smk_ambient_ops, S_IRUGO|S_IWUSR}, [SMK_NET4ADDR] = { @@ -2904,7 +2814,7 @@ static int smk_fill_super(struct super_block *sb, struct fs_context *fc) [SMK_ACCESSES] = { "access", &smk_access_ops, S_IRUGO|S_IWUGO}, [SMK_MAPPED] = { - "mapped", &smk_mapped_ops, S_IRUGO|S_IWUSR}, + "mapped", &smk_cipso_auto_level_ops, 0644}, [SMK_LOAD2] = { "load2", &smk_load2_ops, S_IRUGO|S_IWUSR}, [SMK_LOAD_SELF2] = { diff --git a/sound/core/control_led.c b/sound/core/control_led.c index 8cbacee57ce70b..3d13bbec1c5448 100644 --- a/sound/core/control_led.c +++ b/sound/core/control_led.c @@ -255,6 +255,8 @@ static int snd_ctl_led_set_id(int card_number, struct snd_ctl_elem_id *id, kctl = snd_ctl_find_id(card, id); if (!kctl) return -ENOENT; + if (!kctl->info || !kctl->get) + return -EINVAL; ioff = snd_ctl_get_ioff(kctl, id); vd = &kctl->vd[ioff]; access = vd->access & SNDRV_CTL_ELEM_ACCESS_LED_MASK; diff --git a/sound/core/init.c b/sound/core/init.c index 56dde5bd73c4ef..57733852624a77 100644 --- a/sound/core/init.c +++ b/sound/core/init.c @@ -584,6 +584,8 @@ EXPORT_SYMBOL_GPL(snd_card_disconnect_sync); static int snd_card_do_free(struct snd_card *card) { + bool managed = card->managed; + card->releasing = true; #if IS_ENABLED(CONFIG_SND_MIXER_OSS) if (snd_mixer_oss_notify_callback) @@ -601,7 +603,7 @@ static int snd_card_do_free(struct snd_card *card) } if (card->release_completion) complete(card->release_completion); - if (!card->managed) + if (!managed) kfree(card); return 0; } diff --git a/sound/core/seq/seq_clientmgr.c b/sound/core/seq/seq_clientmgr.c index 28782e1776fa88..9ec6c1c0cabc58 100644 --- a/sound/core/seq/seq_clientmgr.c +++ b/sound/core/seq/seq_clientmgr.c @@ -528,7 +528,7 @@ static int bounce_error_event(struct snd_seq_client *client, struct snd_seq_event *event, int err, int atomic, int hop) { - struct snd_seq_event bounce_ev; + struct snd_seq_event bounce_ev, quoted; int result; if (client == NULL || @@ -548,15 +548,19 @@ static int bounce_error_event(struct snd_seq_client *client, * For user clients, send SNDRV_SEQ_EVENT_BOUNCE with the * original event embedded as variable-length data. This * avoids exposing data.quote.event (a kernel pointer) to - * userspace. The variable-length path in snd_seq_event_dup() - * copies the event data from data.ext.ptr into chained cells, - * and snd_seq_expand_var_event() copies only the data content - * -- never the pointer -- to userspace. + * userspace. Sanitise the embedded copy too - a queued + * variable-length event carries the address of its own + * extension cell, and the payload goes out verbatim. */ + quoted = *event; + if (snd_seq_ev_is_variable("ed)) { + quoted.data.ext.len &= ~SNDRV_SEQ_EXT_MASK; + quoted.data.ext.ptr = NULL; + } bounce_ev.type = SNDRV_SEQ_EVENT_BOUNCE; bounce_ev.flags = SNDRV_SEQ_EVENT_LENGTH_VARIABLE; bounce_ev.data.ext.len = sizeof(struct snd_seq_event); - bounce_ev.data.ext.ptr = (char *)event; + bounce_ev.data.ext.ptr = (char *)"ed; } else { /* * For kernel clients, quote the event pointer directly. diff --git a/sound/core/seq/seq_midi.c b/sound/core/seq/seq_midi.c index 2eb12199c92f90..a16a5debf3391b 100644 --- a/sound/core/seq/seq_midi.c +++ b/sound/core/seq/seq_midi.c @@ -42,9 +42,11 @@ struct seq_midisynth { struct snd_rawmidi *rmidi; int device; int subdevice; + struct snd_rawmidi_substream __rcu *input_substream; + snd_use_lock_t input_use_lock; /* in-flight event_input users */ struct snd_rawmidi_file input_rfile; - spinlock_t output_lock; /* protects output_rfile publication */ snd_use_lock_t output_use_lock; /* in-flight event_input users */ + struct snd_rawmidi_substream __rcu *output_substream; struct snd_rawmidi_file output_rfile; int seq_client; int seq_port; @@ -76,6 +78,14 @@ static void snd_midi_input_event(struct snd_rawmidi_substream *substream) msynth = runtime->private_data; if (msynth == NULL) return; + + scoped_guard(rcu) { + if (rcu_dereference(msynth->input_substream) != substream) + return; + + snd_use_lock_use(&msynth->input_use_lock); + } + memset(&ev, 0, sizeof(ev)); while (runtime->avail > 0) { res = snd_rawmidi_kernel_read(substream, buf, sizeof(buf)); @@ -95,6 +105,8 @@ static void snd_midi_input_event(struct snd_rawmidi_substream *substream) memset(&ev, 0, sizeof(ev)); } } + + snd_use_lock_free(&msynth->input_use_lock); } static int dump_midi(struct snd_rawmidi_substream *substream, const char *buf, int count) @@ -134,8 +146,8 @@ static int event_process_midi(struct snd_seq_event *ev, int direct, if (snd_BUG_ON(!msynth)) return -EINVAL; - scoped_guard(spinlock_irqsave, &msynth->output_lock) { - substream = msynth->output_rfile.output; + scoped_guard(rcu) { + substream = rcu_dereference(msynth->output_substream); if (!substream) return -ENODEV; snd_use_lock_use(&msynth->output_use_lock); @@ -177,7 +189,7 @@ static int snd_seq_midisynth_new(struct seq_midisynth *msynth, msynth->card = card; msynth->device = device; msynth->subdevice = subdevice; - spin_lock_init(&msynth->output_lock); + snd_use_lock_init(&msynth->input_use_lock); snd_use_lock_init(&msynth->output_use_lock); return 0; } @@ -188,28 +200,31 @@ static int midisynth_subscribe(void *private_data, struct snd_seq_port_subscribe int err; struct seq_midisynth *msynth = private_data; struct snd_rawmidi_runtime *runtime; + struct snd_rawmidi_file rfile = {}; struct snd_rawmidi_params params; /* open midi port */ err = snd_rawmidi_kernel_open(msynth->rmidi, msynth->subdevice, SNDRV_RAWMIDI_LFLG_INPUT, - &msynth->input_rfile); + &rfile); if (err < 0) { pr_debug("ALSA: seq_midi: midi input open failed!!!\n"); return err; } - runtime = msynth->input_rfile.input->runtime; + runtime = rfile.input->runtime; memset(¶ms, 0, sizeof(params)); params.avail_min = 1; params.buffer_size = input_buffer_size; - err = snd_rawmidi_input_params(msynth->input_rfile.input, ¶ms); + err = snd_rawmidi_input_params(rfile.input, ¶ms); if (err < 0) { - snd_rawmidi_kernel_release(&msynth->input_rfile); + snd_rawmidi_kernel_release(&rfile); return err; } snd_midi_event_reset_encode(msynth->parser); runtime->event = snd_midi_input_event; runtime->private_data = msynth; + msynth->input_rfile = rfile; + rcu_assign_pointer(msynth->input_substream, rfile.input); snd_rawmidi_kernel_read(msynth->input_rfile.input, NULL, 0); return 0; } @@ -219,10 +234,19 @@ static int midisynth_unsubscribe(void *private_data, struct snd_seq_port_subscri { int err; struct seq_midisynth *msynth = private_data; + struct snd_rawmidi_file rfile; - if (snd_BUG_ON(!msynth->input_rfile.input)) + rcu_assign_pointer(msynth->input_substream, NULL); + synchronize_rcu(); + snd_use_lock_sync(&msynth->input_use_lock); + + rfile = msynth->input_rfile; + msynth->input_rfile = (struct snd_rawmidi_file){}; + + if (snd_BUG_ON(!rfile.input)) return -EINVAL; - err = snd_rawmidi_kernel_release(&msynth->input_rfile); + + err = snd_rawmidi_kernel_release(&rfile); return err; } @@ -252,8 +276,8 @@ static int midisynth_use(void *private_data, struct snd_seq_port_subscribe *info return err; } snd_midi_event_reset_decode(msynth->parser); - scoped_guard(spinlock_irqsave, &msynth->output_lock) - msynth->output_rfile = rfile; + msynth->output_rfile = rfile; + rcu_assign_pointer(msynth->output_substream, rfile.output); return 0; } @@ -261,17 +285,16 @@ static int midisynth_use(void *private_data, struct snd_seq_port_subscribe *info static int midisynth_unuse(void *private_data, struct snd_seq_port_subscribe *info) { struct seq_midisynth *msynth = private_data; - struct snd_rawmidi_file rfile = {}; + struct snd_rawmidi_file rfile; - scoped_guard(spinlock_irqsave, &msynth->output_lock) { - rfile = msynth->output_rfile; - msynth->output_rfile = (struct snd_rawmidi_file){}; - } + rcu_assign_pointer(msynth->output_substream, NULL); + synchronize_rcu(); + snd_use_lock_sync(&msynth->output_use_lock); + rfile = msynth->output_rfile; + msynth->output_rfile = (struct snd_rawmidi_file){}; if (snd_BUG_ON(!rfile.output)) return -EINVAL; - - snd_use_lock_sync(&msynth->output_use_lock); snd_rawmidi_drain_output(rfile.output); return snd_rawmidi_kernel_release(&rfile); } diff --git a/sound/drivers/mtpav.c b/sound/drivers/mtpav.c index d31eadf4be5f83..1ed6d4f0cb625d 100644 --- a/sound/drivers/mtpav.c +++ b/sound/drivers/mtpav.c @@ -642,9 +642,7 @@ static void snd_mtpav_free(struct snd_card *card) { struct mtpav *crd = card->private_data; - guard(spinlock_irqsave)(&crd->spinlock); - if (crd->istimer > 0) - snd_mtpav_remove_output_timer(crd); + timer_shutdown_sync(&crd->timer); } /* diff --git a/sound/hda/codecs/realtek/alc269.c b/sound/hda/codecs/realtek/alc269.c index 3c18f7cae047a6..c31fed171c419c 100644 --- a/sound/hda/codecs/realtek/alc269.c +++ b/sound/hda/codecs/realtek/alc269.c @@ -3524,6 +3524,16 @@ static void alc287_fixup_yoga9_14iap7_bass_spk_pin(struct hda_codec *codec, } } +static void alc_fixup_yoga_pro7_audio(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + /* Reuse the DAC routing selected for ThinkPad X1 Gen7 */ + alc285_fixup_thinkpad_x1_gen7(codec, fix, action); + + /* Limit the internal mic boost to 0 or 1 to avoid noise */ + alc269_fixup_limit_int_mic_boost(codec, fix, action); +} + static void alc295_fixup_dell_inspiron_top_speakers(struct hda_codec *codec, const struct hda_fixup *fix, int action) { @@ -6209,8 +6219,7 @@ static const struct hda_fixup alc269_fixups[] = { }, [ALC287_FIXUP_LENOVO_YOGA_PRO7] = { .type = HDA_FIXUP_FUNC, - /* Reuse the DAC routing selected for ThinkPad X1 Gen7 */ - .v.func = alc285_fixup_thinkpad_x1_gen7, + .v.func = alc_fixup_yoga_pro7_audio, .chained = true, .chain_id = ALC269_FIXUP_LENOVO_XPAD_ACPI, }, diff --git a/sound/hda/common/proc.c b/sound/hda/common/proc.c index c83796b13d3de8..3dabdb418c7bff 100644 --- a/sound/hda/common/proc.c +++ b/sound/hda/common/proc.c @@ -624,7 +624,7 @@ static void print_conn_list(struct snd_info_buffer *buffer, /* Get Cache connections info */ cache_len = snd_hda_get_conn_list(codec, nid, &list); if (cache_len >= 0 && (cache_len != conn_len || - memcmp(list, conn, conn_len) != 0)) { + memcmp(list, conn, conn_len * sizeof(*conn)) != 0)) { snd_iprintf(buffer, " In-driver Connection: %d\n", cache_len); if (cache_len > 0) { snd_iprintf(buffer, " "); diff --git a/sound/hda/controllers/Kconfig b/sound/hda/controllers/Kconfig index 5d6a77e68588e7..26ca69b09a9c3b 100644 --- a/sound/hda/controllers/Kconfig +++ b/sound/hda/controllers/Kconfig @@ -33,6 +33,7 @@ config SND_HDA_TEGRA config SND_HDA_CIX_IPBLOQ tristate "CIX IPBLOQ HD Audio" depends on ARCH_CIX || COMPILE_TEST + depends on ARCH_DMA_ADDR_T_64BIT select SND_HDA select SND_HDA_ALIGNED_MMIO help diff --git a/sound/pci/asihpi/hpi6000.c b/sound/pci/asihpi/hpi6000.c index c8d1518ee3e74d..fd7fe9dba0b807 100644 --- a/sound/pci/asihpi/hpi6000.c +++ b/sound/pci/asihpi/hpi6000.c @@ -537,6 +537,11 @@ static short create_adapter_obj(struct hpi_adapter_obj *pao, hr1.size = sizeof(hr1); error = hpi6000_message_response_sequence(pao, 0, &hm, &hr0); + if (error) { + HPI_DEBUG_LOG(ERROR, "message transport error %d\n", + error); + return error; + } if (hr0.error) { HPI_DEBUG_LOG(DEBUG, "message error %d\n", hr0.error); return hr0.error; diff --git a/sound/pci/ice1712/ice1712.c b/sound/pci/ice1712/ice1712.c index 7d1a357ed90dc3..d6bfd8fb20e65e 100644 --- a/sound/pci/ice1712/ice1712.c +++ b/sound/pci/ice1712/ice1712.c @@ -2523,7 +2523,7 @@ static int snd_ice1712_probe(struct pci_dev *pci, const struct pci_device_id *pci_id) { static int dev; - struct snd_card *card; + struct snd_card *card __free(snd_card_unref) = NULL; struct snd_ice1712 *ice; int pcm_dev = 0, err; const struct snd_ice1712_card_info * const *tbl, *c; @@ -2535,8 +2535,8 @@ static int snd_ice1712_probe(struct pci_dev *pci, return -ENOENT; } - err = snd_card_new(&pci->dev, index[dev], id[dev], THIS_MODULE, - sizeof(*ice), &card); + err = snd_devm_card_new(&pci->dev, index[dev], id[dev], THIS_MODULE, + sizeof(*ice), &card); if (err < 0) return err; ice = card->private_data; @@ -2640,6 +2640,7 @@ static int snd_ice1712_probe(struct pci_dev *pci, if (err < 0) return err; pci_set_drvdata(pci, card); + card = NULL; /* probe succeeded, don't release as error */ dev++; return 0; } diff --git a/sound/pci/via82xx_modem.c b/sound/pci/via82xx_modem.c index 9b84d3fb9eaf5b..b32f84ac17cc1f 100644 --- a/sound/pci/via82xx_modem.c +++ b/sound/pci/via82xx_modem.c @@ -573,24 +573,18 @@ static inline unsigned int calc_linear_pos(struct via82xx_modem *chip, viadev->bufsize2, viadev->idx_table[idx].offset, viadev->idx_table[idx].size, count); #endif - if (count && size < count) { + if (! count) + /* bogus count 0 on the DMA boundary? */ + res = viadev->idx_table[idx].offset; + else + /* count register returns full size + * when end of buffer is reached + */ + res = viadev->idx_table[idx].offset + size; + if (check_invalid_pos(viadev, res)) { dev_dbg(chip->card->dev, - "invalid via82xx_cur_ptr, using last valid pointer\n"); + "invalid via82xx_cur_ptr (2), using last valid pointer\n"); res = viadev->lastpos; - } else { - if (! count) - /* bogus count 0 on the DMA boundary? */ - res = viadev->idx_table[idx].offset; - else - /* count register returns full size - * when end of buffer is reached - */ - res = viadev->idx_table[idx].offset + size; - if (check_invalid_pos(viadev, res)) { - dev_dbg(chip->card->dev, - "invalid via82xx_cur_ptr (2), using last valid pointer\n"); - res = viadev->lastpos; - } } } viadev->lastpos = res; /* remember the last position */ diff --git a/sound/soc/amd/acp-config.c b/sound/soc/amd/acp-config.c index 88e4230d66c3dd..8e1563824a97be 100644 --- a/sound/soc/amd/acp-config.c +++ b/sound/soc/amd/acp-config.c @@ -54,7 +54,7 @@ static const struct dmi_system_id acp70_acpi_flag_override_table[] = { { .matches = { DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC"), - DMI_MATCH(DMI_PRODUCT_NAME, "TUF Gaming A14 FA401EA"), + DMI_MATCH(DMI_PRODUCT_NAME, "FA401EA"), }, }, { diff --git a/sound/soc/apple/mca.c b/sound/soc/apple/mca.c index 492165c0e1ea97..ebe116f32661c0 100644 --- a/sound/soc/apple/mca.c +++ b/sound/soc/apple/mca.c @@ -210,10 +210,10 @@ static void mca_fe_early_trigger(struct snd_pcm_substream *substream, int cmd, SERDES_STATUS_EN | SERDES_STATUS_RST, SERDES_STATUS_RST); /* - * Experiments suggest that it takes at most ~1 us - * for the bit to clear, so wait 2 us for good measure. + * The SERDES cluster needs a bit of time to reset itself + * and settle before we start poking it. This is... slow... */ - udelay(2); + udelay(25); WARN_ON(readl_relaxed(cl->base + serdes_unit + REG_SERDES_STATUS) & SERDES_STATUS_RST); mca_modify(cl, serdes_conf, SERDES_CONF_SYNC_SEL, diff --git a/sound/soc/codecs/rt700-sdw.c b/sound/soc/codecs/rt700-sdw.c index a451d5d1f8ab5a..bb449f08e30cf8 100644 --- a/sound/soc/codecs/rt700-sdw.c +++ b/sound/soc/codecs/rt700-sdw.c @@ -458,10 +458,8 @@ static void rt700_sdw_remove(struct sdw_slave *slave) { struct rt700_priv *rt700 = dev_get_drvdata(&slave->dev); - if (rt700->hw_init) { - cancel_delayed_work_sync(&rt700->jack_detect_work); - cancel_delayed_work_sync(&rt700->jack_btn_check_work); - } + cancel_delayed_work_sync(&rt700->jack_detect_work); + cancel_delayed_work_sync(&rt700->jack_btn_check_work); pm_runtime_disable(&slave->dev); } diff --git a/sound/soc/codecs/tas2783-sdw.c b/sound/soc/codecs/tas2783-sdw.c index e5d27266370e21..4fbbf2ca5bf9ae 100644 --- a/sound/soc/codecs/tas2783-sdw.c +++ b/sound/soc/codecs/tas2783-sdw.c @@ -1098,7 +1098,13 @@ static s32 tas2783_sdca_dev_resume(struct device *dev) } regcache_cache_only(tas_dev->regmap, false); - regcache_sync(tas_dev->regmap); + ret = regcache_sync(tas_dev->regmap); + if (ret) { + regcache_cache_only(tas_dev->regmap, true); + regcache_mark_dirty(tas_dev->regmap); + return ret; + } + return 0; } @@ -1224,9 +1230,23 @@ static s32 tas_update_status(struct sdw_slave *slave, if (tas_dev->hw_init || tas_dev->status != SDW_SLAVE_ATTACHED) return 0; - /* updated the cache data to device */ regcache_cache_only(tas_dev->regmap, false); - regcache_sync(tas_dev->regmap); + + /* + * The device is attaching uninitialized: either this is the first + * attach, or it lost power (and with it all register and DSP state) + * while the controller was power-gated during system suspend. The + * cache still holds the pre-suspend values, and tas_io_init() below + * resets the device via TAS2783_SW_RESET anyway, so syncing it back + * is both useless and harmful: later read-modify-write updates would + * compare against stale data and skip the hardware write. + * + * Drop the cache instead, so that subsequent accesses see the real + * hardware state. Syncing after the reset is not an option either: + * the cache accepts registers for which tas2783_sdca_mbq_size() + * returns 0, and writing those back fails with -EINVAL. + */ + regcache_drop_region(tas_dev->regmap, 0, UINT_MAX); /* perform I/O transfers required for Slave initialization */ return tas_io_init(&slave->dev, slave); diff --git a/sound/soc/codecs/tas675x.c b/sound/soc/codecs/tas675x.c index 82526362de7be0..404706b6215675 100644 --- a/sound/soc/codecs/tas675x.c +++ b/sound/soc/codecs/tas675x.c @@ -1924,15 +1924,6 @@ static const struct reg_default tas675x_reg_defaults[] = { { TAS675X_PWM_PHASE_M_CTRL_CH2_REG, 0x00 }, { TAS675X_PWM_PHASE_M_CTRL_CH3_REG, 0x00 }, { TAS675X_PWM_PHASE_M_CTRL_CH4_REG, 0x00 }, - { TAS675X_DC_LDG_CTRL_REG, 0x00 }, - { TAS675X_DC_LDG_LO_CTRL_REG, 0x00 }, - { TAS675X_DC_LDG_TIME_CTRL_REG, 0x00 }, - { TAS675X_DC_LDG_SL_CH1_CH2_CTRL_REG, 0x11 }, - { TAS675X_DC_LDG_SL_CH3_CH4_CTRL_REG, 0x11 }, - { TAS675X_AC_LDG_CTRL_REG, 0x10 }, - { TAS675X_TWEETER_DETECT_CTRL_REG, 0x08 }, - { TAS675X_TWEETER_DETECT_THRESH_REG, 0x00 }, - { TAS675X_AC_LDG_FREQ_CTRL_REG, 0xC8 }, { TAS675X_REPORT_ROUTING_1_REG, 0x00 }, { TAS675X_OTSD_RECOVERY_EN_REG, 0x00 }, { TAS675X_REPORT_ROUTING_2_REG, 0xA2 }, @@ -1943,6 +1934,15 @@ static const struct reg_default tas675x_reg_defaults[] = { { TAS675X_GPIO1_OUTPUT_SEL_REG, 0x00 }, { TAS675X_GPIO2_OUTPUT_SEL_REG, 0x00 }, { TAS675X_GPIO_CTRL_REG, TAS675X_GPIO_CTRL_RSTVAL }, + { TAS675X_DC_LDG_CTRL_REG, 0x00 }, + { TAS675X_DC_LDG_LO_CTRL_REG, 0x00 }, + { TAS675X_DC_LDG_TIME_CTRL_REG, 0x00 }, + { TAS675X_DC_LDG_SL_CH1_CH2_CTRL_REG, 0x11 }, + { TAS675X_DC_LDG_SL_CH3_CH4_CTRL_REG, 0x11 }, + { TAS675X_AC_LDG_CTRL_REG, 0x10 }, + { TAS675X_TWEETER_DETECT_CTRL_REG, 0x08 }, + { TAS675X_TWEETER_DETECT_THRESH_REG, 0x00 }, + { TAS675X_AC_LDG_FREQ_CTRL_REG, 0xC8 }, { TAS675X_OTW_CTRL_CH1_CH2_REG, 0x11 }, { TAS675X_OTW_CTRL_CH3_CH4_REG, 0x11 }, }; diff --git a/sound/soc/fsl/Kconfig b/sound/soc/fsl/Kconfig index 8ae59c09487880..04940879dfd80b 100644 --- a/sound/soc/fsl/Kconfig +++ b/sound/soc/fsl/Kconfig @@ -244,7 +244,7 @@ config SND_SOC_MPC5200_AC97 config SND_MPC52xx_SOC_PCM030 tristate "SoC AC97 Audio support for Phytec pcm030 and WM9712" depends on PPC_MPC5200_SIMPLE - select SND_SOC_MPC5200_AC97 + depends on SND_SOC_MPC5200_AC97 select SND_SOC_WM9712 help Say Y if you want to add support for sound on the Phytec pcm030 @@ -253,7 +253,7 @@ config SND_MPC52xx_SOC_PCM030 config SND_MPC52xx_SOC_EFIKA tristate "SoC AC97 Audio support for bbplan Efika and STAC9766" depends on PPC_EFIKA - select SND_SOC_MPC5200_AC97 + depends on SND_SOC_MPC5200_AC97 select SND_SOC_STAC9766 help Say Y if you want to add support for sound on the Efika. diff --git a/sound/soc/fsl/efika-audio-fabric.c b/sound/soc/fsl/efika-audio-fabric.c index 4d3266d9cbb79e..e9c728bae8b978 100644 --- a/sound/soc/fsl/efika-audio-fabric.c +++ b/sound/soc/fsl/efika-audio-fabric.c @@ -22,8 +22,6 @@ #include #include -#include "mpc5200_dma.h" - #define DRV_NAME "efika-audio-fabric" SND_SOC_DAILINK_DEFS(analog, diff --git a/sound/soc/fsl/fsl-asoc-card.c b/sound/soc/fsl/fsl-asoc-card.c index 709543308fe9e9..eff46666c8add6 100644 --- a/sound/soc/fsl/fsl-asoc-card.c +++ b/sound/soc/fsl/fsl-asoc-card.c @@ -728,8 +728,8 @@ static int fsl_asoc_card_probe(struct platform_device *pdev) cpu_pdev = of_find_device_by_node(cpu_np); if (!cpu_pdev) { - dev_err(&pdev->dev, "failed to find CPU DAI device\n"); - ret = -EINVAL; + ret = dev_err_probe(&pdev->dev, -EPROBE_DEFER, + "failed to find CPU DAI device\n"); goto fail; } diff --git a/sound/soc/fsl/fsl_audmix.c b/sound/soc/fsl/fsl_audmix.c index f819f33ec46b84..2885cc10b02d45 100644 --- a/sound/soc/fsl/fsl_audmix.c +++ b/sound/soc/fsl/fsl_audmix.c @@ -457,6 +457,9 @@ static const struct of_device_id fsl_audmix_ids[] = { }; MODULE_DEVICE_TABLE(of, fsl_audmix_ids); +static int fsl_audmix_runtime_resume(struct device *dev); +static int fsl_audmix_runtime_suspend(struct device *dev); + static int fsl_audmix_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -488,13 +491,25 @@ static int fsl_audmix_probe(struct platform_device *pdev) spin_lock_init(&priv->lock); platform_set_drvdata(pdev, priv); pm_runtime_enable(dev); + if (!pm_runtime_enabled(dev)) { + ret = fsl_audmix_runtime_resume(dev); + if (ret) + goto err_disable_pm; + } + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + goto err_pm_get_sync; + + /* To enable regmap cache only when runtime PM enabled */ + pm_runtime_put(dev); ret = devm_snd_soc_register_component(dev, &fsl_audmix_component, fsl_audmix_dai, ARRAY_SIZE(fsl_audmix_dai)); if (ret) { dev_err(dev, "failed to register ASoC DAI\n"); - goto err_disable_pm; + goto err_pm_get_sync; } /* @@ -506,12 +521,15 @@ static int fsl_audmix_probe(struct platform_device *pdev) if (IS_ERR(priv->pdev)) { ret = PTR_ERR(priv->pdev); dev_err(dev, "failed to register platform: %d\n", ret); - goto err_disable_pm; + goto err_pm_get_sync; } } return 0; +err_pm_get_sync: + if (!pm_runtime_status_suspended(dev)) + fsl_audmix_runtime_suspend(dev); err_disable_pm: pm_runtime_disable(dev); return ret; @@ -522,6 +540,8 @@ static void fsl_audmix_remove(struct platform_device *pdev) struct fsl_audmix *priv = dev_get_drvdata(&pdev->dev); pm_runtime_disable(&pdev->dev); + if (!pm_runtime_status_suspended(&pdev->dev)) + fsl_audmix_runtime_suspend(&pdev->dev); if (priv->pdev) platform_device_unregister(priv->pdev); diff --git a/sound/soc/fsl/pcm030-audio-fabric.c b/sound/soc/fsl/pcm030-audio-fabric.c index 5542c4ee6d1295..a7a5dba83a2f82 100644 --- a/sound/soc/fsl/pcm030-audio-fabric.c +++ b/sound/soc/fsl/pcm030-audio-fabric.c @@ -13,8 +13,6 @@ #include -#include "mpc5200_dma.h" - #define DRV_NAME "pcm030-audio-fabric" struct pcm030_audio_data { diff --git a/sound/soc/meson/meson-card-utils.c b/sound/soc/meson/meson-card-utils.c index cdb759b466ad43..8617a4661a3392 100644 --- a/sound/soc/meson/meson-card-utils.c +++ b/sound/soc/meson/meson-card-utils.c @@ -50,25 +50,20 @@ int meson_card_reallocate_links(struct snd_soc_card *card, num_links * sizeof(*priv->card.dai_link), GFP_KERNEL | __GFP_ZERO); if (!links) - goto err_links; + return -ENOMEM; + + priv->card.dai_link = links; + priv->card.num_links = num_links; ldata = krealloc(priv->link_data, num_links * sizeof(*priv->link_data), GFP_KERNEL | __GFP_ZERO); + /* meson_card_clean_references() will free the links on this error path */ if (!ldata) - goto err_ldata; + return -ENOMEM; - priv->card.dai_link = links; priv->link_data = ldata; - priv->card.num_links = num_links; return 0; - -err_ldata: - kfree(links); -err_links: - dev_err(priv->card.dev, "failed to allocate links\n"); - return -ENOMEM; - } EXPORT_SYMBOL_GPL(meson_card_reallocate_links); diff --git a/sound/soc/pxa/pxa-ssp.c b/sound/soc/pxa/pxa-ssp.c index f8054c1c59fae7..3a0abcb0bfbdf2 100644 --- a/sound/soc/pxa/pxa-ssp.c +++ b/sound/soc/pxa/pxa-ssp.c @@ -769,13 +769,10 @@ static int pxa_ssp_probe(struct snd_soc_dai *dai) goto err_priv; } - priv->extclk = devm_clk_get(dev, "extclk"); + priv->extclk = devm_clk_get_optional(dev, "extclk"); if (IS_ERR(priv->extclk)) { ret = PTR_ERR(priv->extclk); - if (ret == -EPROBE_DEFER) - goto err_priv; - - priv->extclk = NULL; + goto err_priv; } } else { priv->ssp = pxa_ssp_request(dai->id + 1, "SoC audio"); diff --git a/sound/soc/qcom/qdsp6/q6apm.c b/sound/soc/qcom/qdsp6/q6apm.c index 641d6d24322990..f167b9dae3fa14 100644 --- a/sound/soc/qcom/qdsp6/q6apm.c +++ b/sound/soc/qcom/qdsp6/q6apm.c @@ -802,14 +802,17 @@ EXPORT_SYMBOL_GPL(q6apm_graph_prepare); int q6apm_graph_start(struct q6apm_graph *graph) { struct audioreach_graph *ar_graph = graph->ar_graph; - int ret = 0; + int ret; - if (ar_graph->start_count == 0) + if (ar_graph->start_count == 0) { ret = audioreach_graph_mgmt_cmd(ar_graph, APM_CMD_GRAPH_START); + if (ret) + return ret; + } ar_graph->start_count++; - return ret; + return 0; } EXPORT_SYMBOL_GPL(q6apm_graph_start); @@ -817,6 +820,9 @@ int q6apm_graph_stop(struct q6apm_graph *graph) { struct audioreach_graph *ar_graph = graph->ar_graph; + if (ar_graph->start_count == 0) + return 0; + if (--ar_graph->start_count > 0) return 0; diff --git a/sound/soc/samsung/i2s.c b/sound/soc/samsung/i2s.c index f80f697a5d55d0..f80e8d49815652 100644 --- a/sound/soc/samsung/i2s.c +++ b/sound/soc/samsung/i2s.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -512,7 +513,7 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, u32 mod, mask, val = 0; int ret = 0; - pm_runtime_get_sync(dai->dev); + guard(pm_runtime_active)(dai->dev); scoped_guard(spinlock_irqsave, &priv->lock) mod = readl(priv->addr + I2SMOD); @@ -537,8 +538,7 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, && (mod & cdcon_mask))))) { dev_err(&i2s->pdev->dev, "%s:%d Other DAI busy\n", __func__, __LINE__); - ret = -EAGAIN; - goto err; + return -EAGAIN; } if (dir == SND_SOC_CLOCK_IN) @@ -566,7 +566,7 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, } else { priv->rclk_srcrate = clk_get_rate(priv->op_clk); - goto done; + return 0; } } @@ -580,14 +580,14 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, if (WARN_ON(IS_ERR(priv->op_clk))) { ret = PTR_ERR(priv->op_clk); priv->op_clk = NULL; - goto err; + return ret; } ret = clk_prepare_enable(priv->op_clk); if (ret) { clk_put(priv->op_clk); priv->op_clk = NULL; - goto err; + return ret; } priv->rclk_srcrate = clk_get_rate(priv->op_clk); @@ -595,11 +595,10 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, || (clk_id && !(mod & rsrc_mask))) { dev_err(&i2s->pdev->dev, "%s:%d Other DAI busy\n", __func__, __LINE__); - ret = -EAGAIN; - goto err; + return -EAGAIN; } else { /* Call can't be on the active DAI */ - goto done; + return 0; } if (clk_id == 1) @@ -607,8 +606,7 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, break; default: dev_err(&i2s->pdev->dev, "We don't serve that!\n"); - ret = -EINVAL; - goto err; + return -EINVAL; } scoped_guard(spinlock_irqsave, &priv->lock) { @@ -616,13 +614,8 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, mod = (mod & ~mask) | val; writel(mod, priv->addr + I2SMOD); } -done: - pm_runtime_put(dai->dev); return 0; -err: - pm_runtime_put(dai->dev); - return ret; } static int i2s_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c index 4ad126bd4f7053..646f26b7b4153e 100644 --- a/sound/soc/soc-dapm.c +++ b/sound/soc/soc-dapm.c @@ -3608,7 +3608,7 @@ int snd_soc_dapm_put_enum_double(struct snd_kcontrol *kcontrol, val = snd_soc_enum_item_to_val(e, item[0]) << e->shift_l; mask = e->mask << e->shift_l; if (e->shift_l != e->shift_r) { - if (item[1] > e->items) + if (item[1] >= e->items) return -EINVAL; val |= snd_soc_enum_item_to_val(e, item[1]) << e->shift_r; mask |= e->mask << e->shift_r; diff --git a/sound/soc/sof/ipc4-topology.c b/sound/soc/sof/ipc4-topology.c index 6fdfb667cce8da..8f37fbdd3f7fa2 100644 --- a/sound/soc/sof/ipc4-topology.c +++ b/sound/soc/sof/ipc4-topology.c @@ -1127,6 +1127,7 @@ static int sof_ipc4_widget_setup_comp_src(struct snd_sof_widget *swidget) "Invalid number of formats: input: %d, output: %d\n", src->available_fmt.num_input_formats, src->available_fmt.num_output_formats); + ret = -EINVAL; goto err; } @@ -1179,6 +1180,7 @@ static int sof_ipc4_widget_setup_comp_asrc(struct snd_sof_widget *swidget) "Invalid number of formats: input: %d, output: %d\n", asrc->available_fmt.num_input_formats, asrc->available_fmt.num_output_formats); + ret = -EINVAL; goto err; } diff --git a/sound/soc/sof/topology.c b/sound/soc/sof/topology.c index 42a2d90bb705df..31dd7a66a9cf92 100644 --- a/sound/soc/sof/topology.c +++ b/sound/soc/sof/topology.c @@ -846,6 +846,7 @@ static int sof_control_load_volume(struct snd_soc_component *scomp, struct snd_soc_tplg_mixer_control *mc = container_of(hdr, struct snd_soc_tplg_mixer_control, hdr); int tlv[SOF_TLV_ITEMS]; + u32 min, max; unsigned int mask; int ret; @@ -853,6 +854,11 @@ static int sof_control_load_volume(struct snd_soc_component *scomp, if (le32_to_cpu(mc->num_channels) > SND_SOC_TPLG_MAX_CHAN) return -EINVAL; + min = le32_to_cpu(mc->min); + max = le32_to_cpu(mc->max); + if (min > max || max >= INT_MAX) + return -EINVAL; + /* * If control has more than 2 channels we need to override the info. This is because even if * ASoC layer has defined topology's max channel count to SND_SOC_TPLG_MAX_CHAN = 8, the @@ -863,12 +869,12 @@ static int sof_control_load_volume(struct snd_soc_component *scomp, kc->info = snd_sof_volume_info; scontrol->comp_id = sdev->next_comp_id; - scontrol->min_volume_step = le32_to_cpu(mc->min); - scontrol->max_volume_step = le32_to_cpu(mc->max); + scontrol->min_volume_step = min; + scontrol->max_volume_step = max; scontrol->num_channels = le32_to_cpu(mc->num_channels); - scontrol->max = le32_to_cpu(mc->max); - if (le32_to_cpu(mc->max) == 1) + scontrol->max = max; + if (max == 1) goto skip; /* extract tlv data */ @@ -878,7 +884,7 @@ static int sof_control_load_volume(struct snd_soc_component *scomp, } /* set up volume table */ - ret = set_up_volume_table(scontrol, tlv, le32_to_cpu(mc->max) + 1); + ret = set_up_volume_table(scontrol, tlv, max + 1); if (ret < 0) { dev_err(scomp->dev, "error: setting up volume table\n"); return ret; @@ -911,7 +917,7 @@ skip: return 0; err: - if (le32_to_cpu(mc->max) > 1) + if (max > 1) kfree(scontrol->volume_table); return ret; diff --git a/sound/soc/spacemit/k1_i2s.c b/sound/soc/spacemit/k1_i2s.c index 8871fc15b29cc0..67c03b8a5e0c41 100644 --- a/sound/soc/spacemit/k1_i2s.c +++ b/sound/soc/spacemit/k1_i2s.c @@ -52,7 +52,7 @@ struct spacemit_i2s_dev { struct clk *sysclk; struct clk *bclk; - struct clk *sspa_clk; + struct clk *func_clk; struct clk *sysclk_div; struct clk *c_sysclk; struct clk *c_bclk; @@ -221,7 +221,7 @@ static int spacemit_i2s_hw_params(struct snd_pcm_substream *substream, if (ret) return ret; - return clk_set_rate(i2s->sspa_clk, bclk_rate); + return clk_set_rate(i2s->func_clk, bclk_rate); } static int spacemit_i2s_set_sysclk(struct snd_soc_dai *cpu_dai, int clk_id, @@ -347,22 +347,6 @@ static const struct snd_soc_dai_ops spacemit_i2s_dai_ops = { static struct snd_soc_dai_driver spacemit_i2s_dai = { .ops = &spacemit_i2s_dai_ops, - .playback = { - .channels_min = 1, - .channels_max = 2, - .rates = SPACEMIT_PCM_RATES, - .rate_min = SNDRV_PCM_RATE_8000, - .rate_max = SNDRV_PCM_RATE_48000, - .formats = SPACEMIT_PCM_FORMATS, - }, - .capture = { - .channels_min = 1, - .channels_max = 2, - .rates = SPACEMIT_PCM_RATES, - .rate_min = SNDRV_PCM_RATE_8000, - .rate_max = SNDRV_PCM_RATE_48000, - .formats = SPACEMIT_PCM_FORMATS, - }, .symmetric_rate = 1, }; @@ -392,6 +376,8 @@ static int spacemit_i2s_init_dai(struct spacemit_i2s_dev *i2s, dai->playback.channels_min = 1; dai->playback.channels_max = 2; dai->playback.rates = SPACEMIT_PCM_RATES; + dai->playback.rate_min = SNDRV_PCM_RATE_8000; + dai->playback.rate_max = SNDRV_PCM_RATE_48000; dai->playback.formats = SPACEMIT_PCM_FORMATS; i2s->playback_dma_data.addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; @@ -404,6 +390,8 @@ static int spacemit_i2s_init_dai(struct spacemit_i2s_dev *i2s, dai->capture.channels_min = 1; dai->capture.channels_max = 2; dai->capture.rates = SPACEMIT_PCM_RATES; + dai->capture.rate_min = SNDRV_PCM_RATE_8000; + dai->capture.rate_max = SNDRV_PCM_RATE_48000; dai->capture.formats = SPACEMIT_PCM_FORMATS; i2s->capture_dma_data.addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; @@ -445,14 +433,14 @@ static int spacemit_i2s_probe(struct platform_device *pdev) if (IS_ERR(i2s->bclk)) return dev_err_probe(i2s->dev, PTR_ERR(i2s->bclk), "failed to enable bit clock\n"); - clk = devm_clk_get_enabled(i2s->dev, "sspa_bus"); + clk = devm_clk_get_enabled(i2s->dev, "bus"); if (IS_ERR(clk)) - return dev_err_probe(i2s->dev, PTR_ERR(clk), "failed to enable sspa_bus clock\n"); + return dev_err_probe(i2s->dev, PTR_ERR(clk), "failed to enable bus clock\n"); - i2s->sspa_clk = devm_clk_get_enabled(i2s->dev, "sspa"); - if (IS_ERR(i2s->sspa_clk)) - return dev_err_probe(i2s->dev, PTR_ERR(i2s->sspa_clk), - "failed to enable sspa clock\n"); + i2s->func_clk = devm_clk_get_enabled(i2s->dev, "func"); + if (IS_ERR(i2s->func_clk)) + return dev_err_probe(i2s->dev, PTR_ERR(i2s->func_clk), + "failed to enable func clock\n"); i2s->sysclk_div = devm_clk_get_optional_enabled(i2s->dev, "sysclk_div"); if (IS_ERR(i2s->sysclk_div)) diff --git a/sound/soc/ti/j721e-evm.c b/sound/soc/ti/j721e-evm.c index c214ae0d7b95e3..312298e0b00499 100644 --- a/sound/soc/ti/j721e-evm.c +++ b/sound/soc/ti/j721e-evm.c @@ -4,6 +4,7 @@ * Author: Peter Ujfalusi */ +#include #include #include #include @@ -263,7 +264,7 @@ static int j721e_audio_startup(struct snd_pcm_substream *substream) int ret = 0; int i; - guard(mutex)(&priv->mutex); + mutex_lock(&priv->mutex); domain->active++; @@ -303,6 +304,7 @@ static int j721e_audio_startup(struct snd_pcm_substream *substream) out: if (ret) domain->active--; + mutex_unlock(&priv->mutex); return ret; } diff --git a/sound/soc/xilinx/xlnx_formatter_pcm.c b/sound/soc/xilinx/xlnx_formatter_pcm.c index f15336197ed31e..798f17a6960ac9 100644 --- a/sound/soc/xilinx/xlnx_formatter_pcm.c +++ b/sound/soc/xilinx/xlnx_formatter_pcm.c @@ -384,7 +384,7 @@ static int xlnx_formatter_pcm_open(struct snd_soc_component *component, if (err) { dev_err(component->dev, "Unable to set constraint on period bytes\n"); - return err; + goto error; } /* Resize the buffer bytes as divisible by 64 */ @@ -394,7 +394,7 @@ static int xlnx_formatter_pcm_open(struct snd_soc_component *component, if (err) { dev_err(component->dev, "Unable to set constraint on buffer bytes\n"); - return err; + goto error; } /* Set periods as integer multiple */ @@ -403,7 +403,7 @@ static int xlnx_formatter_pcm_open(struct snd_soc_component *component, if (err < 0) { dev_err(component->dev, "Unable to set constraint on periods to be integer\n"); - return err; + goto error; } /* enable DMA IOC irq */ @@ -412,6 +412,14 @@ static int xlnx_formatter_pcm_open(struct snd_soc_component *component, writel(val, stream_data->mmio + XLNX_AUD_CTRL); return 0; + +error: + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) + adata->play_stream = NULL; + else + adata->capture_stream = NULL; + kfree(stream_data); + return err; } static int xlnx_formatter_pcm_close(struct snd_soc_component *component, diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile index 271a7dc7727309..b0f7168e794329 100644 --- a/tools/bpf/bpftool/Makefile +++ b/tools/bpf/bpftool/Makefile @@ -99,7 +99,7 @@ endif HOST_LDFLAGS := $(LDFLAGS) # Remove warnings for libbpf bootstrap build -LIBBPF_BOOTSTRAP_CFLAGS := $(filter-out -W -Wall -Wextra -Wformat -Wformat-signedness,$(HOST_CFLAGS)) +LIBBPF_BOOTSTRAP_CFLAGS := $(filter-out -W -Wall -Wextra -Wformat%,$(HOST_CFLAGS)) INSTALL ?= install RM ?= rm -f diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c index 71a45d96617edb..684a8fb7241465 100644 --- a/tools/bpf/bpftool/map.c +++ b/tools/bpf/bpftool/map.c @@ -659,8 +659,6 @@ static int do_show_subset(int argc, char **argv) show_map_close_json(fds[i], &info); else show_map_close_plain(fds[i], &info); - - close(fds[i]); } if (json_output && nb_fds > 1) jsonw_end_array(json_wtr); /* root array */ @@ -790,6 +788,12 @@ static int maps_have_btf(int *fds, int nb_fds) static struct btf *btf_vmlinux; +static void free_btf_vmlinux(void) +{ + btf__free(btf_vmlinux); + btf_vmlinux = NULL; +} + static int get_map_kv_btf(const struct bpf_map_info *info, struct btf **btf) { int err = 0; @@ -889,7 +893,6 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr, exit_free: free(key); free(value); - close(fd); free_map_kv_btf(btf); return err; @@ -938,6 +941,7 @@ static int do_dump(int argc, char **argv) for (i = 0; i < nb_fds; i++) { if (bpf_map_get_info_by_fd(fds[i], &info, &len)) { p_err("can't get map info: %s", strerror(errno)); + err = -1; break; } err = map_dump(fds[i], &info, wtr, nb_fds > 1); @@ -958,7 +962,7 @@ exit_close: close(fds[i]); exit_free: free(fds); - btf__free(btf_vmlinux); + free_btf_vmlinux(); return err; } @@ -1049,7 +1053,7 @@ static void print_key_value(struct bpf_map_info *info, void *key, btf_wtr = get_btf_writer(); if (!btf_wtr) { p_info("failed to create json writer for btf. falling back to plain output"); - btf__free(btf); + free_map_kv_btf(btf); btf = NULL; print_entry_plain(info, key, value); } else { @@ -1065,7 +1069,7 @@ static void print_key_value(struct bpf_map_info *info, void *key, } else { print_entry_plain(info, key, value); } - btf__free(btf); + free_map_kv_btf(btf); } static int do_lookup(int argc, char **argv) diff --git a/tools/bpf/bpftool/sign.c b/tools/bpf/bpftool/sign.c index f9b742f4bb104b..1257dba8ef2fd6 100644 --- a/tools/bpf/bpftool/sign.c +++ b/tools/bpf/bpftool/sign.c @@ -175,8 +175,11 @@ int bpftool_prog_sign(struct bpf_load_and_run_opts *opts) goto cleanup; } - EVP_Digest(opts->insns, opts->insns_sz, opts->excl_prog_hash, - &opts->excl_prog_hash_sz, EVP_sha256(), NULL); + if (EVP_Digest(opts->insns, opts->insns_sz, opts->excl_prog_hash, + &opts->excl_prog_hash_sz, EVP_sha256(), NULL) != 1) { + err = -EIO; + goto cleanup; + } bd_out = BIO_new(BIO_s_mem()); if (!bd_out) { diff --git a/tools/bpf/bpftool/struct_ops.c b/tools/bpf/bpftool/struct_ops.c index aa43dead249cba..835e5e561f7fc4 100644 --- a/tools/bpf/bpftool/struct_ops.c +++ b/tools/bpf/bpftool/struct_ops.c @@ -643,6 +643,10 @@ int do_struct_ops(int argc, char **argv) err = cmd_select(cmds, argc, argv, do_help); btf__free(btf_vmlinux); + btf_vmlinux = NULL; + map_info_type = NULL; + map_info_alloc_len = 0; + map_info_type_id = 0; return err; } diff --git a/tools/build/Makefile.feature b/tools/build/Makefile.feature index ed1374af31c168..99eb0ea0953756 100644 --- a/tools/build/Makefile.feature +++ b/tools/build/Makefile.feature @@ -123,9 +123,7 @@ FEATURE_TESTS_EXTRA := \ libbfd-liberty-z \ libopencsd \ libperl \ - cxx \ llvm \ - clang \ libbpf \ libpfm4 \ libdebuginfod \ diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile index 62909a9c799d33..7d165018116a55 100644 --- a/tools/build/feature/Makefile +++ b/tools/build/feature/Makefile @@ -55,7 +55,6 @@ FILES= \ test-bpf.bin \ test-libbpf.bin \ test-sdt.bin \ - test-cxx.bin \ test-gettid.bin \ test-jvmti.bin \ test-jvmti-cmlr.bin \ @@ -63,7 +62,6 @@ FILES= \ test-sched_getcpu.bin \ test-setns.bin \ test-libopencsd.bin \ - test-clang.bin \ test-llvm.bin \ test-llvm-perf.bin \ test-libaio.bin \ @@ -335,9 +333,6 @@ $(OUTPUT)test-libbpf.bin: $(OUTPUT)test-sdt.bin: $(BUILD) -$(OUTPUT)test-cxx.bin: - $(BUILDXX) -std=gnu++11 - $(OUTPUT)test-gettid.bin: $(BUILD) @@ -363,15 +358,6 @@ $(OUTPUT)test-llvm-perf.bin: $(shell $(LLVM_CONFIG) --system-libs) \ > $(@:.bin=.make.output) 2>&1 -$(OUTPUT)test-clang.bin: - $(BUILDXX) -std=gnu++17 \ - -I$(shell $(LLVM_CONFIG) --includedir) \ - -L$(shell $(LLVM_CONFIG) --libdir) \ - -Wl,--start-group -lclang-cpp -Wl,--end-group \ - $(shell $(LLVM_CONFIG) --libs Core option) \ - $(shell $(LLVM_CONFIG) --system-libs) \ - > $(@:.bin=.make.output) 2>&1 - -include $(OUTPUT)*.d $(OUTPUT)test-libaio.bin: diff --git a/tools/include/linux/filter.h b/tools/include/linux/filter.h index bcc6df79301aea..4ead4e72097c13 100644 --- a/tools/include/linux/filter.h +++ b/tools/include/linux/filter.h @@ -6,6 +6,7 @@ #define __TOOLS_LINUX_FILTER_H #include +#include /* ArgX, context and stack frame pointer register positions. Note, * Arg1, Arg2, Arg3, etc are used as argument mappings of function diff --git a/tools/include/nolibc/arch-powerpc.h b/tools/include/nolibc/arch-powerpc.h index a1ab91d5538456..dbe2e5205aaafa 100644 --- a/tools/include/nolibc/arch-powerpc.h +++ b/tools/include/nolibc/arch-powerpc.h @@ -26,7 +26,7 @@ */ #define _NOLIBC_SYSCALL_CLOBBERLIST \ - "memory", "cr0", "r12", "r11", "r10", "r9" + "memory", "cr0", "ctr", "xer", "r12", "r11", "r10", "r9" #define __nolibc_syscall0(num) \ ({ \ diff --git a/tools/include/nolibc/arch-sparc.h b/tools/include/nolibc/arch-sparc.h index ddae9bc10dfe38..23fab40accfad1 100644 --- a/tools/include/nolibc/arch-sparc.h +++ b/tools/include/nolibc/arch-sparc.h @@ -45,7 +45,7 @@ \ __asm__ volatile ( \ _NOLIBC_SYSCALL \ - : "+r"(_arg1) \ + : "=r"(_arg1) \ : "r"(_num) \ : "memory", "cc" \ ); \ diff --git a/tools/include/uapi/linux/if_xdp.h b/tools/include/uapi/linux/if_xdp.h index 23a0627814687e..50d67df78911f5 100644 --- a/tools/include/uapi/linux/if_xdp.h +++ b/tools/include/uapi/linux/if_xdp.h @@ -149,6 +149,7 @@ struct xsk_tx_metadata { __u16 csum_start; /* Offset from csum_start where checksum should be stored. */ __u16 csum_offset; + __u32 reserved; /* XDP_TXMD_FLAGS_LAUNCH_TIME */ /* Launch time in nanosecond against the PTP HW Clock */ diff --git a/tools/lib/api/fd/array.c b/tools/lib/api/fd/array.c index f0f195207fca9b..16a047f1906ee9 100644 --- a/tools/lib/api/fd/array.c +++ b/tools/lib/api/fd/array.c @@ -115,6 +115,9 @@ int fdarray__filter(struct fdarray *fda, short revents, return 0; for (fd = 0; fd < fda->nr; ++fd) { + if (fda->priv[fd].flags & fdarray_flag__nonfilterable) + continue; + if (!fda->entries[fd].events) continue; @@ -122,12 +125,17 @@ int fdarray__filter(struct fdarray *fda, short revents, if (entry_destructor) entry_destructor(fda, fd, arg); + /* + * Set fd to -1 so poll() ignores this entry; otherwise + * POLLHUP/POLLERR are still reported for events=0 fds + * (POSIX: always checked), causing a poll storm. + */ + fda->entries[fd].fd = -1; fda->entries[fd].revents = fda->entries[fd].events = 0; continue; } - if (!(fda->priv[fd].flags & fdarray_flag__nonfilterable)) - ++nr; + ++nr; } return nr; diff --git a/tools/lib/bpf/features.c b/tools/lib/bpf/features.c index b7e388f99d0bbd..e5641fa601637a 100644 --- a/tools/lib/bpf/features.c +++ b/tools/lib/bpf/features.c @@ -577,10 +577,10 @@ static int probe_ldimm64_full_range_off(int token_fd) static int probe_uprobe_syscall(int token_fd) { /* - * If kernel supports uprobe() syscall, it will return -ENXIO when called + * If kernel supports uprobe() syscall, it will return -EPROTO when called * from the outside of a kernel-generated uprobe trampoline. */ - return syscall(__NR_uprobe) < 0 && errno == ENXIO; + return syscall(__NR_uprobe) < 0 && errno == EPROTO; } #else static int probe_uprobe_syscall(int token_fd) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 1368752aa13c39..1ab939dfb7f08c 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -12973,13 +12973,14 @@ static const char *arch_specific_lib_paths(void) /* Get full path to program/shared library. */ static int resolve_full_path(const char *file, char *result, size_t result_sz) { - const char *search_paths[3] = {}; + const char *search_paths[4] = {}; int i, perm; if (str_has_sfx(file, ".so") || strstr(file, ".so.")) { search_paths[0] = getenv("LD_LIBRARY_PATH"); search_paths[1] = "/usr/lib64:/usr/lib"; search_paths[2] = arch_specific_lib_paths(); + search_paths[3] = "/lib64:/lib"; perm = R_OK; } else { search_paths[0] = getenv("PATH"); diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c index 57fb82bb81b588..d2ecd3daab9615 100644 --- a/tools/lib/bpf/usdt.c +++ b/tools/lib/bpf/usdt.c @@ -305,7 +305,7 @@ struct usdt_manager *usdt_manager_new(struct bpf_object *obj) /* * Detect kernel support for uprobe() syscall, it's presence means we can - * take advantage of faster nop5 uprobe handling. + * take advantage of faster nop10 uprobe handling. * Added in: 56101b69c919 ("uprobes/x86: Add uprobe syscall to speed up uprobe") */ man->has_uprobe_syscall = kernel_supports(obj, FEAT_UPROBE_SYSCALL); @@ -604,14 +604,14 @@ static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, #if defined(__x86_64__) static bool has_nop_combo(int fd, long off) { - unsigned char nop_combo[6] = { - 0x90, 0x0f, 0x1f, 0x44, 0x00, 0x00 /* nop,nop5 */ + unsigned char nop_combo[11] = { + 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, }; - unsigned char buf[6]; + unsigned char buf[11]; - if (pread(fd, buf, 6, off) != 6) + if (pread(fd, buf, 11, off) != 11) return false; - return memcmp(buf, nop_combo, 6) == 0; + return memcmp(buf, nop_combo, 11) == 0; } #else static bool has_nop_combo(int fd, long off) @@ -822,8 +822,8 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, memset(target, 0, sizeof(*target)); /* - * We have uprobe syscall and usdt with nop,nop5 instructions combo, - * so we can place the uprobe directly on nop5 (+1) and get this probe + * We have uprobe syscall and usdt with nop,nop10 instructions combo, + * so we can place the uprobe directly on nop10 (+1) and get this probe * optimized. */ if (man->has_uprobe_syscall && has_nop_combo(elf_fd->fd, usdt_rel_ip)) { diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/enum.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/enum.j2 index c7ae506076bb41..d1405c7c5354ae 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/enum.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/enum.j2 @@ -1,3 +1,4 @@ {# SPDX-License-Identifier: GPL-2.0 #} + bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ name }} *ptr); bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, {{ name }} value); diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/string.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/string.j2 index cf65b71eaef396..7ddc2bf3edacc0 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/string.j2 @@ -2,7 +2,9 @@ {% if annotate %} /* member {{ name }} (variable-length string) */ {% endif %} +{% if maxsize != "0" %} if (value->{{ name }}.len > {{ maxsize }}) return false; +{% endif %} if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 index 1d477c2d197aae..5bf00070ae959e 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 @@ -2,7 +2,9 @@ {% if annotate %} /* member {{ name }} (variable-length opaque) */ {% endif %} +{% if maxsize != "0" %} if (value->{{ name }}.len > {{ maxsize }}) return false; +{% endif %} if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/string.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/string.j2 index cf65b71eaef396..7ddc2bf3edacc0 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/string.j2 @@ -2,7 +2,9 @@ {% if annotate %} /* member {{ name }} (variable-length string) */ {% endif %} +{% if maxsize != "0" %} if (value->{{ name }}.len > {{ maxsize }}) return false; +{% endif %} if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 index 1d477c2d197aae..5bf00070ae959e 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 @@ -2,7 +2,9 @@ {% if annotate %} /* member {{ name }} (variable-length opaque) */ {% endif %} +{% if maxsize != "0" %} if (value->{{ name }}.len > {{ maxsize }}) return false; +{% endif %} if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 index 5fc1937ba7744c..19ee759d70c6ab 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 @@ -1,9 +1,3 @@ {# SPDX-License-Identifier: GPL-2.0 #} } u; }; -{%- if name in public_apis %} - - -bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, struct {{ name }} *ptr); -bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const struct {{ name }} *ptr); -{%- endif -%} diff --git a/tools/objtool/Build b/tools/objtool/Build index 93a37b0dfd3130..59f9486280981c 100644 --- a/tools/objtool/Build +++ b/tools/objtool/Build @@ -6,13 +6,15 @@ objtool-y += check.o objtool-y += special.o objtool-y += builtin-check.o objtool-y += elf.o +objtool-y += klp-symid.o objtool-y += objtool.o objtool-$(BUILD_DISAS) += disas.o objtool-$(BUILD_DISAS) += trace.o objtool-$(BUILD_ORC) += orc_gen.o orc_dump.o -objtool-$(BUILD_KLP) += builtin-klp.o klp-checksum.o klp-diff.o klp-post-link.o +objtool-$(BUILD_KLP) += builtin-klp.o klp-checksum.o klp-diff.o \ + klp-post-link.o klp-sympos.o objtool-y += libstring.o objtool-y += libctype.o diff --git a/tools/objtool/arch/x86/special.c b/tools/objtool/arch/x86/special.c index e817a3fff44911..1e84c81bfcd81c 100644 --- a/tools/objtool/arch/x86/special.c +++ b/tools/objtool/arch/x86/special.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include #include #include #include @@ -9,6 +10,32 @@ /* cpu feature name array generated from cpufeatures.h */ #include "cpu-feature-names.c" +/* + * An alternative with an empty replacement, e.g. the second entry of + * + * ALTERNATIVE_2("orig", "repl", ft1, "", ft2) + * + * still gets a relocation for its replacement offset. But the label it points + * at is the end of the previous entry's replacement, which is also the + * beginning of the *next* entry's replacement. The value is meaningless: it's + * only ever used with a length of zero. + */ +bool arch_alt_ignore_new_reloc(struct section *sec, unsigned long offset) +{ + unsigned long entry_off; + + if (strcmp(sec->name, ".altinstructions")) + return false; + + entry_off = offset - (offset % ALT_ENTRY_SIZE); + + if (offset - entry_off != ALT_NEW_OFFSET) + return false; + + return !*(unsigned char *)(sec->data->d_buf + entry_off + + ALT_NEW_LEN_OFFSET); +} + void arch_handle_alternative(struct special_alt *alt) { static struct special_alt *group, *prev; diff --git a/tools/objtool/builtin-check.c b/tools/objtool/builtin-check.c index 118c3de2f293ee..75b11dc85010e1 100644 --- a/tools/objtool/builtin-check.c +++ b/tools/objtool/builtin-check.c @@ -76,6 +76,7 @@ static const struct option check_options[] = { OPT_STRING_OPTARG('d', "disas", &opts.disas, "function-pattern", "disassemble functions", "*"), OPT_CALLBACK_OPTARG('h', "hacks", NULL, NULL, "jump_label,noinstr,skylake", "patch toolchain bugs/limitations", parse_hacks), OPT_BOOLEAN('i', "ibt", &opts.ibt, "validate and annotate IBT"), + OPT_BOOLEAN(0, "klp-symids", &opts.klp_symids, "generate .klp.symids for duplicate symbol disambiguation"), OPT_BOOLEAN('m', "mcount", &opts.mcount, "annotate mcount/fentry calls for ftrace"), OPT_BOOLEAN(0, "noabs", &opts.noabs, "reject absolute references in allocatable sections"), OPT_BOOLEAN('n', "noinstr", &opts.noinstr, "validate noinstr rules"), @@ -174,10 +175,16 @@ static bool opts_valid(void) return false; } + if (opts.klp_symids && !opts.link) { + ERROR("--klp-symids requires --link"); + return false; + } + if (opts.disas || opts.hack_jump_label || opts.hack_noinstr || opts.ibt || + opts.klp_symids || opts.mcount || opts.noabs || opts.noinstr || diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 3ab5b9f1c6a42f..28cc2fed6f15e6 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -4925,6 +4926,12 @@ int check(struct objtool_file *file) goto out; } + if (opts.klp_symids) { + ret = klp_create_symid_sections(file); + if (ret) + goto out; + } + if (opts.noabs) warnings += check_abs_references(file); diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c index 33c95a74a51bd8..a791f4ea6ec192 100644 --- a/tools/objtool/elf.c +++ b/tools/objtool/elf.c @@ -23,6 +23,7 @@ #include #include #include +#include #include static ssize_t demangled_name_len(const char *name); @@ -626,6 +627,18 @@ static int read_symbols(struct elf *elf) return -1; } + /* + * "klp diff" renames the placeholder symbols of KLP relocs to + * hide them from modpost. Hide the prefix from the rest of + * objtool so its many name-based heuristics (noreturns, + * uaccess safe list, ...) still see the original symbol name. + * + * st_name is left alone, so the renamed symbol is preserved in + * the output file. + */ + if (strstarts(sym->name, KLP_TOMBSTONE_PREFIX)) + sym->name += strlen(KLP_TOMBSTONE_PREFIX); + if ((sym->sym.st_shndx > SHN_UNDEF && sym->sym.st_shndx < SHN_LORESERVE) || (shndx_data && sym->sym.st_shndx == SHN_XINDEX)) { diff --git a/tools/objtool/include/objtool/builtin.h b/tools/objtool/include/objtool/builtin.h index e844e9c82b7b2e..349690bb1c50eb 100644 --- a/tools/objtool/include/objtool/builtin.h +++ b/tools/objtool/include/objtool/builtin.h @@ -16,6 +16,7 @@ struct opts { bool hack_noinstr; bool hack_skylake; bool ibt; + bool klp_symids; bool mcount; bool noabs; bool noinstr; diff --git a/tools/objtool/include/objtool/elf.h b/tools/objtool/include/objtool/elf.h index d9c44df9cc76a4..a82517a76a0f64 100644 --- a/tools/objtool/include/objtool/elf.h +++ b/tools/objtool/include/objtool/elf.h @@ -97,6 +97,7 @@ struct symbol { u8 included : 1; u8 klp : 1; u8 dont_correlate : 1; + u8 fake : 1; struct list_head pv_target; struct reloc *relocs; struct section *group_sec; diff --git a/tools/objtool/include/objtool/klp.h b/tools/objtool/include/objtool/klp.h index 6f60cf05db8644..c57775d78c71e4 100644 --- a/tools/objtool/include/objtool/klp.h +++ b/tools/objtool/include/objtool/klp.h @@ -14,21 +14,48 @@ #define KLP_FUNCS_SEC ".init.klp_funcs" /* - * __klp_relocs is an intermediate section which are created by klp diff and - * converted into KLP symbols/relas by "objtool klp post-link". This is needed - * to work around the linker, which doesn't preserve SHN_LIVEPATCH or + * __klp_relocs. are intermediate sections which are created by klp + * diff and converted into KLP symbols/relas by "objtool klp post-link". This + * is needed to work around the linker, which doesn't preserve SHN_LIVEPATCH or * SHF_RELA_LIVEPATCH, nor does it support having two RELA sections for a * single PROGBITS section. + * + * "objname" is the object whose loading gates the relocation: "vmlinux" for + * references to vmlinux symbols, otherwise the name of the module being + * patched. post-link uses it to name the resulting + * .klp.rela.objname.section_name sections. */ #define KLP_RELOCS_SEC "__klp_relocs" #define KLP_STRINGS_SEC ".rodata.klp.str1.1" +#define KLP_TOMBSTONE_PREFIX ".klp.tombstone." + struct klp_reloc { void *offset; void *sym; u32 type; }; +/* + * .klp.symid is used to correlate symbols between vmlinux.o and vmlinux, for + * calculating sympos to disambiguate duplicately-named symbols. + */ +#define KLP_SYMID_SEC ".klp.symid" + +struct klp_symid { + u64 id; + u64 addr; +}; + +struct objtool_file; +struct elf; +struct symbol; + +int klp_create_symid_sections(struct objtool_file *file); + +int klp_sympos_init(struct elf *orig); +unsigned long klp_find_sympos(struct elf *elf, struct symbol *sym); + int cmd_klp_checksum(int argc, const char **argv); int cmd_klp_diff(int argc, const char **argv); int cmd_klp_post_link(int argc, const char **argv); diff --git a/tools/objtool/include/objtool/special.h b/tools/objtool/include/objtool/special.h index 121c3761899c14..620dbf6cb0e581 100644 --- a/tools/objtool/include/objtool/special.h +++ b/tools/objtool/include/objtool/special.h @@ -32,6 +32,13 @@ int special_get_alts(struct elf *elf, struct list_head *alts); void arch_handle_alternative(struct special_alt *alt); +/* + * Should the reloc at @offset -- the "new" (replacement) field of a special + * section group entry -- be ignored? The meaning of a zero-length replacement + * is arch specific, so the arch decides. + */ +bool arch_alt_ignore_new_reloc(struct section *sec, unsigned long offset); + bool arch_support_alt_relocation(struct special_alt *special_alt, struct instruction *insn, struct reloc *reloc); diff --git a/tools/objtool/klp-diff.c b/tools/objtool/klp-diff.c index f8787d7d14547f..291ada0aa288d1 100644 --- a/tools/objtool/klp-diff.c +++ b/tools/objtool/klp-diff.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include @@ -30,7 +30,9 @@ struct elfs { struct export { struct hlist_node hash; - char *mod, *sym; + char *mod; + char *sym; + bool mod_ns; }; bool debug, debug_correlate, debug_clone; @@ -83,6 +85,35 @@ static char *escape_str(const char *orig) return new; } +/* + * Convert a build-tree object path to a runtime module name: strip + * directory components, replace '-' with '_', and remove file + * extensions. Examples: + * + * "arch/x86/kvm/kvm" -> "kvm" + * "arch/x86/kvm/kvm-intel" -> "kvm_intel". + * + * Used by read_exports() to normalize Module.symvers entries and by + * __find_modname() as a fallback when .modinfo lacks a "name=" tag. + */ +static char *normalize_modname(char *name) +{ + char *slash = strrchr(name, '/'); + + if (slash) + name = slash + 1; + + for (char *c = name; *c; c++) { + if (*c == '-') + *c = '_'; + else if (*c == '.') { + *c = '\0'; + break; + } + } + return name; +} + static int read_exports(void) { const char *symvers = "Module.symvers"; @@ -106,7 +137,7 @@ static int read_exports(void) } while (fgets(line, 1024, file)) { - char *sym, *mod, *type; + char *sym, *mod, *type, *namespace; struct export *export; sym = strchr(line, '\t'); @@ -133,6 +164,14 @@ static int read_exports(void) *type++ = '\0'; + namespace = strchr(type, '\t'); + if (!namespace) { + ERROR("malformed Module.symvers (namespace) at line %d", line_num); + return -1; + } + + *namespace++ = '\0'; + if (*sym == '\0' || *mod == '\0') { ERROR("malformed Module.symvers at line %d", line_num); return -1; @@ -150,12 +189,18 @@ static int read_exports(void) return -1; } + if (strcmp(export->mod, "vmlinux")) + export->mod = normalize_modname(export->mod); + export->sym = strdup(sym); if (!export->sym) { ERROR_GLIBC("strdup"); return -1; } + /* EXPORT_SYMBOL_FOR_MODULES() */ + export->mod_ns = strstarts(namespace, "module:"); + hash_add(exports, &export->hash, str_hash(sym)); } @@ -866,65 +911,6 @@ static int correlate_symbols(struct elfs *e) return 0; } -/* "sympos" is used by livepatch to disambiguate duplicate symbol names */ -static unsigned long find_sympos(struct elf *elf, struct symbol *sym) -{ - bool vmlinux = str_ends_with(objname, "vmlinux.o"); - unsigned long sympos = 0, nr_matches = 0; - bool has_dup = false; - struct symbol *s; - - if (sym->bind != STB_LOCAL) - return 0; - - if (vmlinux && is_func_sym(sym)) { - /* - * HACK: Unfortunately, symbol ordering can differ between - * vmlinux.o and vmlinux due to the linker script emitting - * .text.unlikely* before .text*. Count .text.unlikely* first. - * - * TODO: Disambiguate symbols more reliably (checksums?) - */ - for_each_sym(elf, s) { - if (strstarts(s->sec->name, ".text.unlikely") && - !strcmp(s->name, sym->name)) { - nr_matches++; - if (s == sym) - sympos = nr_matches; - else - has_dup = true; - } - } - for_each_sym(elf, s) { - if (!strstarts(s->sec->name, ".text.unlikely") && - !strcmp(s->name, sym->name)) { - nr_matches++; - if (s == sym) - sympos = nr_matches; - else - has_dup = true; - } - } - } else { - for_each_sym(elf, s) { - if (!strcmp(s->name, sym->name)) { - nr_matches++; - if (s == sym) - sympos = nr_matches; - else - has_dup = true; - } - } - } - - if (!sympos) { - ERROR("can't find sympos for %s", sym->name); - return ULONG_MAX; - } - - return has_dup ? sympos : 0; -} - static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym); static struct symbol *__clone_symbol(struct elf *elf, struct symbol *patched_sym, @@ -1129,6 +1115,9 @@ static struct export *find_export(struct symbol *sym) { struct export *export; + if (is_local_sym(sym)) + return NULL; + hash_for_each_possible(exports, export, hash, str_hash(sym->name)) { if (!strcmp(export->sym, sym->name)) return export; @@ -1158,18 +1147,7 @@ static const char *__find_modname(struct elfs *e) return NULL; } - for (char *c = name; *c; c++) { - if (*c == '/') - name = c + 1; - else if (*c == '-') - *c = '_'; - else if (*c == '.') { - *c = '\0'; - break; - } - } - - return name; + return normalize_modname(name); } /* Get the object's module name as defined by the kernel (and klp_object) */ @@ -1210,11 +1188,16 @@ static bool klp_reloc_needed(struct reloc *patched_reloc) * clusterfunk that is late module patching, the patch module is * allowed to be loaded before any modules it depends on. * - * If exported by vmlinux, a normal reloc will do. + * If exported by vmlinux to all modules, a normal reloc will do. */ export = find_export(patched_sym); - if (export) - return strcmp(export->mod, "vmlinux"); + if (export) { + if (strcmp(export->mod, "vmlinux")) + return true; + + /* EXPORT_SYMBOL_FOR_MODULES() gets a klp reloc */ + return export->mod_ns; + } if (!patched_sym->twin) { /* @@ -1337,12 +1320,14 @@ static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc, struct section *sec, unsigned long offset, struct export *export) { + const char *sym_modname, *sym_orig_name, *sec_objname; struct symbol *patched_sym = patched_reloc->sym; s64 addend = reloc_addend(patched_reloc); - const char *sym_modname, *sym_orig_name; - static struct section *klp_relocs; + char tombstone_name[SYM_NAME_LEN]; struct symbol *sym, *klp_sym; unsigned long klp_reloc_off; + struct section *klp_relocs; + char sec_name[SEC_NAME_LEN]; char sym_name[SYM_NAME_LEN]; struct klp_reloc klp_reloc; unsigned long sympos; @@ -1355,15 +1340,22 @@ static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc, /* * Keep the original reloc intact for now to avoid breaking objtool run * which relies on proper relocations for many of its features. This - * will be disabled later by "objtool klp post-link". + * reloc now targets a functionally dead tombstone symbol and will be + * disabled later by "objtool klp post-link". * - * Convert it to UNDEF (and WEAK to avoid modpost warnings). + * Convert the symbol to UNDEF/WEAK and rename to + * .klp.tombstone.sym_name to prevent modpost from printing warnings or + * creating false module dependencies. The prefix is hidden from the + * objtool run itself by read_symbols(). */ sym = patched_sym->clone; if (!sym) { - /* STB_WEAK: avoid modpost undefined symbol warnings */ - sym = elf_create_symbol(e->out, patched_sym->name, NULL, + if (snprintf_check(tombstone_name, SYM_NAME_LEN, + KLP_TOMBSTONE_PREFIX "%s", patched_sym->name)) + return -1; + + sym = elf_create_symbol(e->out, tombstone_name, NULL, STB_WEAK, patched_sym->type, 0, 0); if (!sym) return -1; @@ -1389,7 +1381,7 @@ static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc, return -1; sym_orig_name = patched_sym->twin->name; - sympos = find_sympos(e->orig, patched_sym->twin); + sympos = klp_find_sympos(e->orig, patched_sym->twin); if (sympos == ULONG_MAX) return -1; } @@ -1411,16 +1403,35 @@ static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc, } /* - * Create the __klp_relocs entry. This will be converted to an actual - * KLP rela by "objtool klp post-link". + * Create the __klp_relocs. entry. This will be converted to + * an actual KLP rela by "objtool klp post-link". * * This intermediate step is necessary to prevent corruption by the * linker, which doesn't know how to properly handle two rela sections * applying to the same base section. + * + * The objname decides when the reloc gets applied. A reference to a + * vmlinux symbol goes in the vmlinux section so it gets applied when + * the patch module loads. Everything else goes in the patched + * object's section, applied when the patched module is loaded. */ + if (!strcmp(sym_modname, "vmlinux")) { + sec_objname = "vmlinux"; + } else { + sec_objname = find_modname(e); + if (!sec_objname) + return -1; + } + + /* section format: __klp_relocs.objname */ + if (snprintf_check(sec_name, SEC_NAME_LEN, + KLP_RELOCS_SEC ".%s", sec_objname)) + return -1; + + klp_relocs = find_section_by_name(e->out, sec_name); if (!klp_relocs) { - klp_relocs = elf_create_section(e->out, KLP_RELOCS_SEC, 0, + klp_relocs = elf_create_section(e->out, sec_name, 0, 0, SHT_PROGBITS, 8, SHF_ALLOC); if (!klp_relocs) return -1; @@ -1568,6 +1579,10 @@ static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym) !strcmp(patched_reloc->sym->sec->name, ".altinstr_aux")) continue; + if (arch_alt_ignore_new_reloc(patched_sym->sec, + reloc_offset(patched_reloc))) + continue; + ret = convert_reloc_sym(e->patched, patched_reloc); if (ret < 0) { ERROR_FUNC(patched_rsec->base, reloc_offset(patched_reloc), @@ -1591,6 +1606,7 @@ static int create_fake_symbol(struct elf *elf, struct section *sec, unsigned long offset, size_t size) { char name[SYM_NAME_LEN]; + struct symbol *sym; unsigned int type; static int ctr; char *c; @@ -1607,7 +1623,24 @@ static int create_fake_symbol(struct elf *elf, struct section *sec, * while still allowing objdump to disassemble it. */ type = is_text_sec(sec) ? STT_NOTYPE : STT_OBJECT; - return elf_create_symbol(elf, name, sec, STB_LOCAL, type, offset, size) ? 0 : -1; + + sym = elf_create_symbol(elf, name, sec, STB_LOCAL, type, offset, size); + if (!sym) + return -1; + + sym->fake = 1; + return 0; +} + +static bool has_fake_symbols(struct section *sec) +{ + struct symbol *sym; + + sec_for_each_sym(sec, sym) + if (sym->fake) + return true; + + return false; } /* @@ -1658,13 +1691,17 @@ static int create_fake_symbols(struct elf *elf) for_each_reloc(sec->rsec, reloc) { unsigned long offset, size; struct reloc *next_reloc; + bool last = true; if (annotype(elf, sec, reloc) != ANNOTYPE_DATA_SPECIAL) continue; offset = reloc_addend(reloc); - size = 0; + /* + * Find the start of the next entry so the fake symbol size can + * be calculated. + */ next_reloc = reloc; for_each_reloc_continue(sec->rsec, next_reloc) { if (annotype(elf, sec, next_reloc) != ANNOTYPE_DATA_SPECIAL || @@ -1672,10 +1709,15 @@ static int create_fake_symbols(struct elf *elf) continue; size = reloc_addend(next_reloc) - offset; + last = false; break; } - if (!size) + /* + * If no next entry found, this is the last entry, so its size + * is from the current offset to the end of the section. + */ + if (last) size = sec_size(reloc->sym->sec) - offset; if (create_fake_symbol(elf, reloc->sym->sec, offset, size)) @@ -1690,7 +1732,11 @@ entsize: unsigned int entry_size; unsigned long offset; - if (!is_special_section(sec) || find_symbol_by_offset(sec, 0)) + if (!is_special_section(sec)) + continue; + + /* Skip sections already handled by step 1 above */ + if (has_fake_symbols(sec)) continue; if (!sec->rsec) { @@ -2007,7 +2053,7 @@ static int create_klp_sections(struct elfs *e) /* klp_func_ext.sympos */ BUILD_BUG_ON(sizeof(sympos) != sizeof_field(struct klp_func_ext, sympos)); - sympos = find_sympos(e->orig, sym->clone->twin); + sympos = klp_find_sympos(e->orig, sym->clone->twin); if (sympos == ULONG_MAX) return -1; memcpy(func_data + offsetof(struct klp_func_ext, sympos), &sympos, @@ -2161,6 +2207,9 @@ int cmd_klp_diff(int argc, const char **argv) if (!e.orig || !e.patched) return -1; + if (klp_sympos_init(e.orig)) + return -1; + if (read_exports()) return -1; diff --git a/tools/objtool/klp-post-link.c b/tools/objtool/klp-post-link.c index c013e39957b117..350d20495897b6 100644 --- a/tools/objtool/klp-post-link.c +++ b/tools/objtool/klp-post-link.c @@ -19,19 +19,11 @@ #include #include -static int fix_klp_relocs(struct elf *elf) +static int fix_klp_reloc_sec(struct elf *elf, struct section *symtab, + struct section *klp_relocs) { - struct section *symtab, *klp_relocs; - - klp_relocs = find_section_by_name(elf, KLP_RELOCS_SEC); - if (!klp_relocs) - return 0; - - symtab = find_section_by_name(elf, ".symtab"); - if (!symtab) { - ERROR("missing .symtab"); - return -1; - } + /* section format: __klp_relocs.sec_objname */ + const char *sec_objname = klp_relocs->name + strlen(KLP_RELOCS_SEC "."); for (int i = 0; i < sec_size(klp_relocs) / sizeof(struct klp_reloc); i++) { struct klp_reloc *klp_reloc; @@ -39,7 +31,6 @@ static int fix_klp_relocs(struct elf *elf) struct section *sec, *tmp, *klp_rsec; unsigned long offset; struct reloc *reloc; - char sym_modname[64]; char rsec_name[SEC_NAME_LEN]; u64 addend; struct symbol *sym, *klp_sym; @@ -55,7 +46,7 @@ static int fix_klp_relocs(struct elf *elf) reloc = find_reloc_by_dest(elf, klp_relocs, klp_reloc_off + offsetof(struct klp_reloc, offset)); if (!reloc) { - ERROR("malformed " KLP_RELOCS_SEC " section"); + ERROR("malformed %s section", klp_relocs->name); return -1; } @@ -66,17 +57,13 @@ static int fix_klp_relocs(struct elf *elf) reloc = find_reloc_by_dest(elf, klp_relocs, klp_reloc_off + offsetof(struct klp_reloc, sym)); if (!reloc) { - ERROR("malformed " KLP_RELOCS_SEC " section"); + ERROR("malformed %s section", klp_relocs->name); return -1; } klp_sym = reloc->sym; addend = reloc_addend(reloc); - /* symbol format: .klp.sym.modname.sym_name,sympos */ - if (sscanf(klp_sym->name + strlen(KLP_SYM_PREFIX), "%55[^.]", sym_modname) != 1) - ERROR("can't find modname in klp symbol '%s'", klp_sym->name); - /* * Create the KLP rela: */ @@ -84,7 +71,7 @@ static int fix_klp_relocs(struct elf *elf) /* section format: .klp.rela.sec_objname.section_name */ if (snprintf_check(rsec_name, SEC_NAME_LEN, KLP_RELOC_SEC_PREFIX "%s.%s", - sym_modname, sec->name)) + sec_objname, sec->name)) return -1; klp_rsec = find_section_by_name(elf, rsec_name); @@ -134,10 +121,32 @@ static int fix_klp_relocs(struct elf *elf) return 0; } +static int fix_klp_relocs(struct elf *elf) +{ + struct section *symtab, *sec; + + symtab = find_section_by_name(elf, ".symtab"); + if (!symtab) { + ERROR("missing .symtab"); + return -1; + } + + for_each_sec(elf, sec) { + if (strncmp(sec->name, KLP_RELOCS_SEC ".", + strlen(KLP_RELOCS_SEC "."))) + continue; + + if (fix_klp_reloc_sec(elf, symtab, sec)) + return -1; + } + + return 0; +} + /* * This runs on the livepatch module after all other linking has been done. It - * converts the intermediate __klp_relocs section into proper KLP relocs to be - * processed by livepatch. This needs to run last to avoid linker wreckage. + * converts the intermediate __klp_relocs.* sections into proper KLP relocs to + * be processed by livepatch. This needs to run last to avoid linker wreckage. * Linkers don't tend to handle the "two rela sections for a single base * section" case very well, nor do they appreciate SHN_LIVEPATCH. */ diff --git a/tools/objtool/klp-symid.c b/tools/objtool/klp-symid.c new file mode 100644 index 00000000000000..b19f76dff13e95 --- /dev/null +++ b/tools/objtool/klp-symid.c @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Emit the .klp.symid table which allows "objtool klp diff" to reliably + * disambiguate duplicate-named local symbols in vmlinux. + * + * Livepatch identifies a duplicate-named symbol by its position (sympos) + * among the same-named kallsyms entries, counted in ascending address order + * in the final linked vmlinux. That order can't be derived from vmlinux.o + * alone: the final link reorders sub-sections (.text.unlikely*, .data..*, + * etc). + * + * Bridge the gap with a table which survives the final link: a single + * non-alloc section containing an array of { id, addr } entries, where + * 'id' is a unique counter identifier and 'addr' has a relocation to the + * symbol. The linker copies 'id' verbatim and resolves 'addr' to the symbol's + * final address. + * + * The table is only emitted for vmlinux.o, and only when klp-build asks for it + * with KLP_SYMIDS=1, which adds --klp-symids to the vmlinux.o objtool run. + * + * It can't survive --gc-sections, which sweeps the whole section; klp-build + * rejects CONFIG_LD_DEAD_CODE_DATA_ELIMINATION. + */ +#include + +#include +#include +#include +#include + +static const char * const discarded_secs[] = { + ".discard", + ".exitcall.exit", + ".modinfo", + ".no_trim_symbol", + "__tracepoint_check", +}; + +static bool discarded_sec(struct section *sec) +{ + if (!(sec->sh.sh_flags & SHF_ALLOC)) + return true; + + for (int i = 0; i < ARRAY_SIZE(discarded_secs); i++) + if (strstarts(sec->name, discarded_secs[i])) + return true; + + return false; +} + +static bool symid_needed(struct elf *elf, struct symbol *sym) +{ + struct symbol *s; + + if (!is_local_sym(sym) || is_undef_sym(sym)) + return false; + + if (!is_func_sym(sym) && !is_object_sym(sym)) + return false; + + if (is_prefix_func(sym)) + return false; + + if (discarded_sec(sym->sec)) + return false; + + for_each_sym_by_name(elf, sym->name, s) { + if (s == sym || is_sec_sym(s) || is_file_sym(s) || is_undef_sym(s)) + continue; + return true; + } + + return false; +} + +int klp_create_symid_sections(struct objtool_file *file) +{ + struct elf *elf = file->elf; + struct klp_symid *symids; + struct section *sec; + struct symbol *sym; + u64 nr = 0, i = 0; + + if (!str_ends_with(objname, "vmlinux.o")) + return 0; + + for_each_sym(elf, sym) + if (symid_needed(elf, sym)) + nr++; + + if (!nr) + return 0; + + sec = elf_create_section(elf, KLP_SYMID_SEC, 0, sizeof(struct klp_symid), + SHT_PROGBITS, 8, 0); + if (!sec) + return -1; + + symids = elf_add_data(elf, sec, NULL, nr * sizeof(struct klp_symid)); + if (!symids) + return -1; + + for_each_sym(elf, sym) { + if (!symid_needed(elf, sym)) + continue; + + symids[i].id = bswap_if_needed(elf, i); + + if (!elf_create_reloc(elf, sec, + i * sizeof(struct klp_symid) + + offsetof(struct klp_symid, addr), + sym, 0, R_ABS64)) + return -1; + + i++; + } + + return 0; +} diff --git a/tools/objtool/klp-sympos.c b/tools/objtool/klp-sympos.c new file mode 100644 index 00000000000000..bbfae516d33951 --- /dev/null +++ b/tools/objtool/klp-sympos.c @@ -0,0 +1,411 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Compute "sympos", the position used by livepatch to disambiguate + * duplicate symbol names in the patched object. + */ +#include +#include +#include + +#include +#include +#include +#include + +#include + +struct vmlinux_sym { + struct hlist_node hash; + const char *name; + u64 addr; +}; + +struct vmlinux_symid { + struct hlist_node hash; + u64 id; + u64 addr; +}; + +struct vmlinux_o_symid { + struct hlist_node hash; + u64 id; + unsigned int sym_idx; +}; + +static DEFINE_HASHTABLE(vmlinux_o_symids, 16); + +/* + * The original linked kernel, found next to the orig vmlinux.o. Read with raw + * libelf rather than elf_open_read(): only the symbol table and the resolved + * .klp.symid table are needed, not the (huge) instruction/reloc machinery. + * + * Both tables are built once by read_orig_vmlinux(). The Elf handle stays + * open because the hashed names point into its mmapped string table. + */ +static struct { + Elf *elf; + DECLARE_HASHTABLE(syms, 16); /* name -> address */ + DECLARE_HASHTABLE(symids, 16); /* .klp.symid id -> address */ +} vmlinux; + +/* + * Would the symbol be visible to the runtime's kallsyms-based symbol lookup? + */ +static bool vmlinux_sym_in_kallsyms(Elf *elf, GElf_Sym *sym) +{ + unsigned int type = GELF_ST_TYPE(sym->st_info); + GElf_Shdr shdr; + Elf_Scn *scn; + + if (sym->st_shndx == SHN_UNDEF || sym->st_shndx >= SHN_LORESERVE) + return false; + + if (type == STT_SECTION || type == STT_FILE) + return false; + + scn = elf_getscn(elf, sym->st_shndx); + if (!scn || !gelf_getshdr(scn, &shdr)) + return false; + + return shdr.sh_flags & SHF_ALLOC; +} + +static int read_orig_vmlinux(const char *filename) +{ + size_t shstrndx, nr_syms = 0, nr_symids = 0, strtab_idx = 0; + Elf_Data *symtab_data = NULL, *symid_data = NULL; + struct klp_symid *symids; + Elf_Scn *scn = NULL; + GElf_Ehdr ehdr; + int fd; + + fd = open(filename, O_RDONLY); + if (fd == -1) { + ERROR_GLIBC("can't open '%s'", filename); + return -1; + } + + if (elf_version(EV_CURRENT) == EV_NONE) { + ERROR_ELF("elf_version"); + return -1; + } + + vmlinux.elf = elf_begin(fd, ELF_C_READ_MMAP, NULL); + if (!vmlinux.elf) { + ERROR_ELF("elf_begin"); + return -1; + } + + if (!gelf_getehdr(vmlinux.elf, &ehdr)) { + ERROR_ELF("gelf_getehdr"); + return -1; + } + + if (elf_getshdrstrndx(vmlinux.elf, &shstrndx)) { + ERROR_ELF("elf_getshdrstrndx"); + return -1; + } + + while ((scn = elf_nextscn(vmlinux.elf, scn))) { + const char *name; + GElf_Shdr shdr; + + if (!gelf_getshdr(scn, &shdr)) { + ERROR_ELF("gelf_getshdr"); + return -1; + } + + if (shdr.sh_type == SHT_SYMTAB) { + symtab_data = elf_getdata(scn, NULL); + if (!symtab_data) { + ERROR_ELF("elf_getdata"); + return -1; + } + nr_syms = shdr.sh_size / shdr.sh_entsize; + strtab_idx = shdr.sh_link; + continue; + } + + name = elf_strptr(vmlinux.elf, shstrndx, shdr.sh_name); + if (name && !strcmp(name, KLP_SYMID_SEC)) { + if (shdr.sh_size % sizeof(struct klp_symid)) { + ERROR("%s: %s: struct klp_symid size mismatch", + filename, KLP_SYMID_SEC); + return -1; + } + symid_data = elf_getdata(scn, NULL); + if (!symid_data) { + ERROR_ELF("elf_getdata"); + return -1; + } + nr_symids = shdr.sh_size / sizeof(struct klp_symid); + } + } + + if (!symtab_data) { + ERROR("%s: missing symbol table", filename); + return -1; + } + + if (!symid_data) { + ERROR("%s: missing %s section, kernel not built with CONFIG_KLP_BUILD?", + filename, KLP_SYMID_SEC); + return -1; + } + + for (size_t i = 0; i < nr_syms; i++) { + struct vmlinux_sym *vsym; + const char *name; + GElf_Sym s; + + if (!gelf_getsym(symtab_data, i, &s)) { + ERROR_ELF("gelf_getsym"); + return -1; + } + + if (!vmlinux_sym_in_kallsyms(vmlinux.elf, &s)) + continue; + + name = elf_strptr(vmlinux.elf, strtab_idx, s.st_name); + if (!name) + continue; + + vsym = calloc(1, sizeof(*vsym)); + if (!vsym) { + ERROR_GLIBC("calloc"); + return -1; + } + + vsym->name = name; + vsym->addr = s.st_value; + hash_add(vmlinux.syms, &vsym->hash, str_hash(name)); + } + + symids = symid_data->d_buf; + + for (size_t i = 0; i < nr_symids; i++) { + struct vmlinux_symid *vsymid; + + vsymid = calloc(1, sizeof(*vsymid)); + if (!vsymid) { + ERROR_GLIBC("calloc"); + return -1; + } + + vsymid->id = __bswap_if_needed(&ehdr, symids[i].id); + vsymid->addr = __bswap_if_needed(&ehdr, symids[i].addr); + hash_add(vmlinux.symids, &vsymid->hash, vsymid->id); + } + + /* the fd and Elf handle stay open, the hashed names live in the mmap */ + return 0; +} + +/* + * Read the orig vmlinux.o's .klp.symid table, an array of entries whose 'addr' + * fields have relocs to the symbols they describe. + */ +static int read_vmlinux_o_symids(struct elf *vmlinux_o) +{ + struct section *sec; + + for_each_sec(vmlinux_o, sec) { + unsigned long nr; + + if (strcmp(sec->name, KLP_SYMID_SEC)) + continue; + + if (sec_size(sec) % sizeof(struct klp_symid)) { + ERROR("%s: %s: struct klp_symid size mismatch", + vmlinux_o->name, KLP_SYMID_SEC); + return -1; + } + + nr = sec_size(sec) / sizeof(struct klp_symid); + + for (unsigned long i = 0; i < nr; i++) { + unsigned long offset = i * sizeof(struct klp_symid); + struct vmlinux_o_symid *entry; + struct klp_symid *symid; + struct reloc *reloc; + + entry = calloc(1, sizeof(*entry)); + if (!entry) { + ERROR_GLIBC("calloc"); + return -1; + } + + symid = sec->data->d_buf + offset; + entry->id = bswap_if_needed(vmlinux_o, symid->id); + + reloc = find_reloc_by_dest(vmlinux_o, sec, + offset + offsetof(struct klp_symid, addr)); + if (!reloc) { + ERROR("%s: missing reloc for %s entry", + vmlinux_o->name, KLP_SYMID_SEC); + return -1; + } + entry->sym_idx = reloc->sym->idx; + + hash_add(vmlinux_o_symids, &entry->hash, entry->sym_idx); + } + } + + return 0; +} + +int klp_sympos_init(struct elf *orig) +{ + char *filename; + int ret; + + if (!str_ends_with(objname, "vmlinux.o")) + return 0; + + if (read_vmlinux_o_symids(orig)) + return -1; + + filename = strndup(objname, strlen(objname) - 2); + if (!filename) { + ERROR_GLIBC("strndup"); + return -1; + } + + ret = read_orig_vmlinux(filename); + free(filename); + + return ret; +} + +/* Find the symbol's id in the orig vmlinux.o's .klp.symid table */ +static int find_vmlinux_o_symid(struct symbol *sym, u64 *id) +{ + struct vmlinux_o_symid *entry; + + hash_for_each_possible(vmlinux_o_symids, entry, hash, sym->idx) { + if (entry->sym_idx == sym->idx) { + *id = entry->id; + return 0; + } + } + + ERROR("no %s entry for symbol %s in orig vmlinux.o", KLP_SYMID_SEC, + sym->name); + return -1; +} + +/* Find the symbol's final address in the orig vmlinux's .klp.symid table */ +static int find_vmlinux_symid_addr(u64 id, u64 *addr) +{ + struct vmlinux_symid *symid; + + hash_for_each_possible(vmlinux.symids, symid, hash, id) { + if (symid->id == id) { + *addr = symid->addr; + return 0; + } + } + + return -1; +} + +/* + * Find the sympos of a vmlinux-local symbol by ranking its final address + * among the duplicately named symbols in the linked orig vmlinux, replicating + * the order in which kallsyms_on_each_match_symbol() counts them. + */ +static unsigned long find_vmlinux_sympos(struct symbol *sym) +{ + unsigned long nr_matches = 0, sympos = 1; + u32 key = str_hash(sym->name); + struct vmlinux_sym *vsym; + bool found = false; + u64 id, addr; + + hash_for_each_possible(vmlinux.syms, vsym, hash, key) + if (!strcmp(vsym->name, sym->name)) + nr_matches++; + + if (!nr_matches) { + ERROR("can't find symbol %s in orig vmlinux", sym->name); + return ULONG_MAX; + } + + /* + * Unique symbols don't need disambiguating. They also have no + * .klp.symid entry, which is only emitted for names duplicated in + * vmlinux.o, so the lookups below would fail. + */ + if (nr_matches == 1) + return 0; + + if (find_vmlinux_o_symid(sym, &id)) + return ULONG_MAX; + + if (find_vmlinux_symid_addr(id, &addr)) { + ERROR("no %s entry for symbol %s in orig vmlinux", KLP_SYMID_SEC, + sym->name); + return ULONG_MAX; + } + + hash_for_each_possible(vmlinux.syms, vsym, hash, key) { + if (strcmp(vsym->name, sym->name)) + continue; + + if (vsym->addr < addr) + sympos++; + else if (vsym->addr == addr) + found = true; + } + + if (!found) { + ERROR("%s address mismatch for symbol %s, stale orig vmlinux?", + KLP_SYMID_SEC, sym->name); + return ULONG_MAX; + } + + return sympos; +} + +/* + * "sympos" is used by livepatch to disambiguate duplicate symbol names. + */ +unsigned long klp_find_sympos(struct elf *elf, struct symbol *sym) +{ + unsigned long sympos = 0, nr_matches = 0; + bool has_dup = false; + struct symbol *s; + + if (sym->bind != STB_LOCAL) + return 0; + + /* + * vmlinux: the final link reorders symbols relative to vmlinux.o, + * so the position needs to be derived from the linked orig vmlinux via + * the .klp.symid table. + */ + if (vmlinux.elf) + return find_vmlinux_sympos(sym); + + /* + * modules: the final .ko preserves symbol table order, so a + * symtab-order count here matches the runtime count done by + * module_kallsyms_on_each_symbol(). + */ + for_each_sym(elf, s) { + if (!strcmp(s->name, sym->name)) { + nr_matches++; + if (s == sym) + sympos = nr_matches; + else + has_dup = true; + } + } + + if (!sympos) { + ERROR("can't find sympos for %s", sym->name); + return ULONG_MAX; + } + + return has_dup ? sympos : 0; +} diff --git a/tools/perf/arch/arm/util/cs-etm.c b/tools/perf/arch/arm/util/cs-etm.c index cdf8e3e6060670..d2861d66a6612e 100644 --- a/tools/perf/arch/arm/util/cs-etm.c +++ b/tools/perf/arch/arm/util/cs-etm.c @@ -201,7 +201,7 @@ static int cs_etm_validate_config(struct perf_pmu *cs_etm_pmu, { unsigned int idx; int err = 0; - struct perf_cpu_map *event_cpus = evsel->evlist->core.user_requested_cpus; + struct perf_cpu_map *event_cpus = evlist__core(evsel->evlist)->user_requested_cpus; struct perf_cpu_map *intersect_cpus; struct perf_cpu cpu; @@ -325,7 +325,7 @@ static int cs_etm_recording_options(struct auxtrace_record *itr, container_of(itr, struct cs_etm_recording, itr); struct perf_pmu *cs_etm_pmu = ptr->cs_etm_pmu; struct evsel *evsel, *cs_etm_evsel = NULL; - struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; + struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus; bool privileged = perf_event_paranoid_check(-1); int err = 0; @@ -551,7 +551,7 @@ cs_etm_info_priv_size(struct auxtrace_record *itr, { unsigned int idx; int etmv3 = 0, etmv4 = 0, ete = 0; - struct perf_cpu_map *event_cpus = evlist->core.user_requested_cpus; + struct perf_cpu_map *event_cpus = evlist__core(evlist)->user_requested_cpus; struct perf_cpu_map *intersect_cpus; struct perf_cpu cpu; struct perf_pmu *cs_etm_pmu = cs_etm_get_pmu(itr); @@ -790,7 +790,7 @@ static int cs_etm_info_fill(struct auxtrace_record *itr, u32 offset; u64 nr_cpu, type; struct perf_cpu_map *cpu_map; - struct perf_cpu_map *event_cpus = session->evlist->core.user_requested_cpus; + struct perf_cpu_map *event_cpus = evlist__core(session->evlist)->user_requested_cpus; struct perf_cpu_map *online_cpus = perf_cpu_map__new_online_cpus(); struct cs_etm_recording *ptr = container_of(itr, struct cs_etm_recording, itr); @@ -800,7 +800,7 @@ static int cs_etm_info_fill(struct auxtrace_record *itr, if (priv_size != cs_etm_info_priv_size(itr, session->evlist)) return -EINVAL; - if (!session->evlist->core.nr_mmaps) + if (!evlist__core(session->evlist)->nr_mmaps) return -EINVAL; /* If the cpu_map has the "any" CPU all online CPUs are involved */ diff --git a/tools/perf/arch/arm64/util/arm-spe.c b/tools/perf/arch/arm64/util/arm-spe.c index 91bb28cad79a55..1ba803a8d9b45a 100644 --- a/tools/perf/arch/arm64/util/arm-spe.c +++ b/tools/perf/arch/arm64/util/arm-spe.c @@ -60,7 +60,7 @@ static bool arm_spe_is_set_freq(struct evsel *evsel) */ static struct perf_cpu_map *arm_spe_find_cpus(struct evlist *evlist) { - struct perf_cpu_map *event_cpus = evlist->core.user_requested_cpus; + struct perf_cpu_map *event_cpus = evlist__core(evlist)->user_requested_cpus; struct perf_cpu_map *online_cpus = perf_cpu_map__new_online_cpus(); struct perf_cpu_map *intersect_cpus; @@ -157,7 +157,7 @@ static int arm_spe_info_fill(struct auxtrace_record *itr, if (priv_size != arm_spe_info_priv_size(itr, session->evlist)) return -EINVAL; - if (!session->evlist->core.nr_mmaps) + if (!evlist__core(session->evlist)->nr_mmaps) return -EINVAL; cpu_map = arm_spe_find_cpus(session->evlist); @@ -363,7 +363,7 @@ static int arm_spe_setup_tracking_event(struct evlist *evlist, { int err; struct evsel *tracking_evsel; - struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; + struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus; /* Add dummy event to keep tracking */ err = parse_event(evlist, "dummy:u"); @@ -396,7 +396,7 @@ static int arm_spe_recording_options(struct auxtrace_record *itr, struct arm_spe_recording *sper = container_of(itr, struct arm_spe_recording, itr); struct evsel *evsel, *tmp; - struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; + struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus; bool discard = false; int err; u64 discard_bit; diff --git a/tools/perf/arch/arm64/util/hisi-ptt.c b/tools/perf/arch/arm64/util/hisi-ptt.c index fe457fd58c9e80..52257715d2b74d 100644 --- a/tools/perf/arch/arm64/util/hisi-ptt.c +++ b/tools/perf/arch/arm64/util/hisi-ptt.c @@ -53,7 +53,7 @@ static int hisi_ptt_info_fill(struct auxtrace_record *itr, if (priv_size != HISI_PTT_AUXTRACE_PRIV_SIZE) return -EINVAL; - if (!session->evlist->core.nr_mmaps) + if (!evlist__core(session->evlist)->nr_mmaps) return -EINVAL; auxtrace_info->type = PERF_AUXTRACE_HISI_PTT; diff --git a/tools/perf/arch/x86/tests/amd-ibs-period.c b/tools/perf/arch/x86/tests/amd-ibs-period.c index 6a92b3a23ed7a6..32713f8fcd5c85 100644 --- a/tools/perf/arch/x86/tests/amd-ibs-period.c +++ b/tools/perf/arch/x86/tests/amd-ibs-period.c @@ -46,7 +46,6 @@ static int dummy_workload_1(unsigned long count) { int (*func)(void); int ret = 0; - char *p; char insn1[] = { 0xb8, 0x01, 0x00, 0x00, 0x00, /* mov 1,%eax */ 0xc3, /* ret */ @@ -59,18 +58,11 @@ static int dummy_workload_1(unsigned long count) 0xcc, /* int 3 */ }; - p = calloc(2, page_size); - if (!p) { - printf("malloc() failed. %m"); - return 1; - } - - func = (void *)((unsigned long)(p + page_size - 1) & ~(page_size - 1)); - - ret = mprotect(func, page_size, PROT_READ | PROT_WRITE | PROT_EXEC); - if (ret) { - printf("mprotect() failed. %m"); - goto out; + func = mmap(NULL, page_size, PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (func == MAP_FAILED) { + pr_debug("mmap() failed. %m\n"); + return -1; } if (count < 100000) @@ -93,7 +85,7 @@ static int dummy_workload_1(unsigned long count) } out: - free(p); + munmap(func, page_size); return ret; } diff --git a/tools/perf/arch/x86/tests/hybrid.c b/tools/perf/arch/x86/tests/hybrid.c index e221ea10417406..0477e17b8e53df 100644 --- a/tools/perf/arch/x86/tests/hybrid.c +++ b/tools/perf/arch/x86/tests/hybrid.c @@ -26,7 +26,7 @@ static int test__hybrid_hw_event_with_pmu(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 1 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 1 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW)); TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES)); @@ -38,7 +38,7 @@ static int test__hybrid_hw_group_event(struct evlist *evlist) struct evsel *evsel, *leader; evsel = leader = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW)); TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES)); @@ -57,7 +57,7 @@ static int test__hybrid_sw_hw_group_event(struct evlist *evlist) struct evsel *evsel, *leader; evsel = leader = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_SOFTWARE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong leader", evsel__has_leader(evsel, leader)); @@ -74,7 +74,7 @@ static int test__hybrid_hw_sw_group_event(struct evlist *evlist) struct evsel *evsel, *leader; evsel = leader = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW)); TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES)); @@ -91,7 +91,7 @@ static int test__hybrid_group_modifier1(struct evlist *evlist) struct evsel *evsel, *leader; evsel = leader = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW)); TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES)); @@ -113,7 +113,7 @@ static int test__hybrid_raw1(struct evlist *evlist) { struct perf_evsel *evsel; - perf_evlist__for_each_evsel(&evlist->core, evsel) { + perf_evlist__for_each_evsel(evlist__core(evlist), evsel) { struct perf_pmu *pmu = perf_pmus__find_by_type(evsel->attr.type); TEST_ASSERT_VAL("missing pmu", pmu); @@ -127,7 +127,7 @@ static int test__hybrid_raw2(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 1 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 1 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_RAW == evsel->core.attr.type); TEST_ASSERT_VAL("wrong config", test_config(evsel, 0x1a)); return TEST_OK; @@ -137,7 +137,7 @@ static int test__hybrid_cache_event(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 1 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 1 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_HW_CACHE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong config", 0x2 == (evsel->core.attr.config & 0xffffffff)); return TEST_OK; @@ -148,7 +148,7 @@ static int test__checkevent_pmu(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 1 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 1 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_RAW == evsel->core.attr.type); TEST_ASSERT_VAL("wrong config", 10 == evsel->core.attr.config); TEST_ASSERT_VAL("wrong config1", 1 == evsel->core.attr.config1); @@ -168,7 +168,7 @@ static int test__hybrid_hw_group_event_2(struct evlist *evlist) struct evsel *evsel, *leader; evsel = leader = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW)); TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES)); @@ -268,7 +268,7 @@ static int test_event(const struct evlist_test *e) ret = e->check(evlist); } parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); return ret; } diff --git a/tools/perf/arch/x86/tests/topdown.c b/tools/perf/arch/x86/tests/topdown.c index 221f2c4bbb615b..2b6f47ce493247 100644 --- a/tools/perf/arch/x86/tests/topdown.c +++ b/tools/perf/arch/x86/tests/topdown.c @@ -56,7 +56,7 @@ static int event_cb(void *state, struct pmu_event_info *info) *ret = TEST_FAIL; } } - evlist__delete(evlist); + evlist__put(evlist); return 0; } @@ -174,7 +174,7 @@ static int test_sort(const char *str, int expected_slots_group_size, CHECK_COND(slots_seen, "slots seen"); ret = TEST_OK; out_err: - evlist__delete(evlist); + evlist__put(evlist); parse_events_error__exit(&err); return ret; } diff --git a/tools/perf/arch/x86/util/auxtrace.c b/tools/perf/arch/x86/util/auxtrace.c index ecbf61a7eb3a3e..84fce0b51ccf78 100644 --- a/tools/perf/arch/x86/util/auxtrace.c +++ b/tools/perf/arch/x86/util/auxtrace.c @@ -55,7 +55,7 @@ struct auxtrace_record *auxtrace_record__init(struct evlist *evlist, int *err) { char buffer[64]; - struct perf_cpu cpu = perf_cpu_map__min(evlist->core.all_cpus); + struct perf_cpu cpu = perf_cpu_map__min(evlist__core(evlist)->all_cpus); int ret; *err = 0; diff --git a/tools/perf/arch/x86/util/intel-bts.c b/tools/perf/arch/x86/util/intel-bts.c index 100a23d27998f0..d44d568a6d2108 100644 --- a/tools/perf/arch/x86/util/intel-bts.c +++ b/tools/perf/arch/x86/util/intel-bts.c @@ -79,10 +79,10 @@ static int intel_bts_info_fill(struct auxtrace_record *itr, if (priv_size != INTEL_BTS_AUXTRACE_PRIV_SIZE) return -EINVAL; - if (!session->evlist->core.nr_mmaps) + if (!evlist__core(session->evlist)->nr_mmaps) return -EINVAL; - pc = session->evlist->mmap[0].core.base; + pc = evlist__mmap(session->evlist)[0].core.base; if (pc) { err = perf_read_tsc_conversion(pc, &tc); if (err) { @@ -114,7 +114,7 @@ static int intel_bts_recording_options(struct auxtrace_record *itr, container_of(itr, struct intel_bts_recording, itr); struct perf_pmu *intel_bts_pmu = btsr->intel_bts_pmu; struct evsel *evsel, *intel_bts_evsel = NULL; - const struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; + const struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus; bool privileged = perf_event_paranoid_check(-1); if (opts->auxtrace_sample_mode) { diff --git a/tools/perf/arch/x86/util/intel-pt.c b/tools/perf/arch/x86/util/intel-pt.c index 0307ff15d9fc90..a533114c0048cb 100644 --- a/tools/perf/arch/x86/util/intel-pt.c +++ b/tools/perf/arch/x86/util/intel-pt.c @@ -360,10 +360,10 @@ static int intel_pt_info_fill(struct auxtrace_record *itr, filter = intel_pt_find_filter(session->evlist, ptr->intel_pt_pmu); filter_str_len = filter ? strlen(filter) : 0; - if (!session->evlist->core.nr_mmaps) + if (!evlist__core(session->evlist)->nr_mmaps) return -EINVAL; - pc = session->evlist->mmap[0].core.base; + pc = evlist__mmap(session->evlist)[0].core.base; if (pc) { err = perf_read_tsc_conversion(pc, &tc); if (err) { @@ -376,7 +376,8 @@ static int intel_pt_info_fill(struct auxtrace_record *itr, ui__warning("Intel Processor Trace: TSC not available\n"); } - per_cpu_mmaps = !perf_cpu_map__is_any_cpu_or_is_empty(session->evlist->core.user_requested_cpus); + per_cpu_mmaps = !perf_cpu_map__is_any_cpu_or_is_empty( + evlist__core(session->evlist)->user_requested_cpus); auxtrace_info->type = PERF_AUXTRACE_INTEL_PT; auxtrace_info->priv[INTEL_PT_PMU_TYPE] = intel_pt_pmu->type; @@ -621,7 +622,7 @@ static int intel_pt_recording_options(struct auxtrace_record *itr, struct perf_pmu *intel_pt_pmu = ptr->intel_pt_pmu; bool have_timing_info, need_immediate = false; struct evsel *evsel, *intel_pt_evsel = NULL; - const struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; + const struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus; bool privileged = perf_event_paranoid_check(-1); u64 tsc_bit; int err; diff --git a/tools/perf/arch/x86/util/iostat.c b/tools/perf/arch/x86/util/iostat.c index 7442a2cd87eda7..b13abea3a6f480 100644 --- a/tools/perf/arch/x86/util/iostat.c +++ b/tools/perf/arch/x86/util/iostat.c @@ -332,13 +332,15 @@ err: return ret; } -int iostat_prepare(struct evlist *evlist, struct perf_stat_config *config) +int iostat_prepare(struct evlist **evlist_ptr, struct perf_stat_config *config) { - if (evlist->core.nr_entries > 0) { + struct evlist *evlist = *evlist_ptr; + + if (evlist__nr_entries(evlist) > 0) { pr_warning("The -e and -M options are not supported." "All chosen events/metrics will be dropped\n"); - evlist__delete(evlist); - evlist = evlist__new(); + evlist__put(evlist); + *evlist_ptr = evlist = evlist__new(); if (!evlist) return -ENOMEM; } @@ -400,7 +402,7 @@ void iostat_prefix(struct evlist *evlist, struct perf_stat_config *config, char *prefix, struct timespec *ts) { - struct iio_root_port *rp = evlist->selected->priv; + struct iio_root_port *rp = evlist__selected(evlist)->priv; if (rp) { /* @@ -463,7 +465,7 @@ void iostat_print_counters(struct evlist *evlist, iostat_prefix(evlist, config, prefix, ts); fprintf(config->output, "%s", prefix); evlist__for_each_entry(evlist, counter) { - perf_device = evlist->selected->priv; + perf_device = evlist__selected(evlist)->priv; if (perf_device && perf_device != counter->priv) { evlist__set_selected(evlist, counter); iostat_prefix(evlist, config, prefix, ts); diff --git a/tools/perf/bench/evlist-open-close.c b/tools/perf/bench/evlist-open-close.c index faf9c34b4a5dc0..748ebbe458f493 100644 --- a/tools/perf/bench/evlist-open-close.c +++ b/tools/perf/bench/evlist-open-close.c @@ -76,7 +76,7 @@ static struct evlist *bench__create_evlist(char *evstr, const char *uid_str) parse_events_error__exit(&err); pr_err("Run 'perf list' for a list of valid events\n"); ret = 1; - goto out_delete_evlist; + goto out_put_evlist; } parse_events_error__exit(&err); if (uid_str) { @@ -85,24 +85,24 @@ static struct evlist *bench__create_evlist(char *evstr, const char *uid_str) if (uid == UINT_MAX) { pr_err("Invalid User: %s", uid_str); ret = -EINVAL; - goto out_delete_evlist; + goto out_put_evlist; } ret = parse_uid_filter(evlist, uid); if (ret) - goto out_delete_evlist; + goto out_put_evlist; } ret = evlist__create_maps(evlist, &opts.target); if (ret < 0) { pr_err("Not enough memory to create thread/cpu maps\n"); - goto out_delete_evlist; + goto out_put_evlist; } evlist__config(evlist, &opts, NULL); return evlist; -out_delete_evlist: - evlist__delete(evlist); +out_put_evlist: + evlist__put(evlist); return NULL; } @@ -116,7 +116,7 @@ static int bench__do_evlist_open_close(struct evlist *evlist) return err; } - err = evlist__mmap(evlist, opts.mmap_pages); + err = evlist__do_mmap(evlist, opts.mmap_pages); if (err < 0) { pr_err("evlist__mmap: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); return err; @@ -124,7 +124,7 @@ static int bench__do_evlist_open_close(struct evlist *evlist) evlist__enable(evlist); evlist__disable(evlist); - evlist__munmap(evlist); + evlist__do_munmap(evlist); evlist__close(evlist); return 0; @@ -145,13 +145,14 @@ static int bench_evlist_open_close__run(char *evstr, const char *uid_str) init_stats(&time_stats); - printf(" Number of cpus:\t%d\n", perf_cpu_map__nr(evlist->core.user_requested_cpus)); - printf(" Number of threads:\t%d\n", evlist->core.threads->nr); + printf(" Number of cpus:\t%d\n", + perf_cpu_map__nr(evlist__core(evlist)->user_requested_cpus)); + printf(" Number of threads:\t%d\n", evlist__core(evlist)->threads->nr); printf(" Number of events:\t%d (%d fds)\n", - evlist->core.nr_entries, evlist__count_evsel_fds(evlist)); + evlist__nr_entries(evlist), evlist__count_evsel_fds(evlist)); printf(" Number of iterations:\t%d\n", iterations); - evlist__delete(evlist); + evlist__put(evlist); for (i = 0; i < iterations; i++) { pr_debug("Started iteration %d\n", i); @@ -162,7 +163,7 @@ static int bench_evlist_open_close__run(char *evstr, const char *uid_str) gettimeofday(&start, NULL); err = bench__do_evlist_open_close(evlist); if (err) { - evlist__delete(evlist); + evlist__put(evlist); return err; } @@ -171,7 +172,7 @@ static int bench_evlist_open_close__run(char *evstr, const char *uid_str) runtime_us = timeval2usec(&diff); update_stats(&time_stats, runtime_us); - evlist__delete(evlist); + evlist__put(evlist); pr_debug("Iteration %d took:\t%" PRIu64 "us\n", i, runtime_us); } diff --git a/tools/perf/builtin-annotate.c b/tools/perf/builtin-annotate.c index 8a0eb30eac24fd..69cb72b2082a67 100644 --- a/tools/perf/builtin-annotate.c +++ b/tools/perf/builtin-annotate.c @@ -562,7 +562,7 @@ static int __cmd_annotate(struct perf_annotate *ann) goto out; if ((use_browser == 1 || ann->use_stdio2) && ann->has_br_stack) - if (session->evlist->nr_br_cntr > 0) + if (evlist__nr_br_cntr(session->evlist) > 0) annotate_opts.show_br_cntr = true; if (dump_trace) { @@ -928,8 +928,11 @@ int cmd_annotate(int argc, const char **argv) * branch counters, if the corresponding branch info is available * in the perf data in the TUI mode. */ - if ((use_browser == 1 || annotate.use_stdio2) && annotate.has_br_stack) + if ((use_browser == 1 || annotate.use_stdio2) && annotate.has_br_stack) { sort__mode = SORT_MODE__BRANCH; + if (evlist__nr_br_cntr(annotate.session->evlist) > 0) + annotate_opts.show_br_cntr = true; + } if (setup_sorting(/*evlist=*/NULL, perf_session__env(annotate.session)) < 0) usage_with_options(annotate_usage, options); diff --git a/tools/perf/builtin-c2c.c b/tools/perf/builtin-c2c.c index c9584dbedf77af..bc16a57e092742 100644 --- a/tools/perf/builtin-c2c.c +++ b/tools/perf/builtin-c2c.c @@ -12,11 +12,14 @@ */ #include #include +#include +#include #include #include #include #include +#include #include #include #include @@ -226,7 +229,6 @@ he__get_c2c_hists(struct hist_entry *he, ret = c2c_hists__init(hists, sort, nr_header_lines, env); if (ret) { - perf_hpp__reset_output_field(&hists->list); c2c_he->hists = NULL; free(hists); return NULL; @@ -2063,26 +2065,38 @@ static int c2c_hists__init_sort(struct perf_hpp_list *hpp_list, char *name, stru return 0; } -#define PARSE_LIST(_list, _fn) \ - do { \ - char *tmp, *tok; \ - ret = 0; \ - \ - if (!_list) \ - break; \ - \ - for (tok = strtok_r((char *)_list, ", ", &tmp); \ - tok; tok = strtok_r(NULL, ", ", &tmp)) { \ - ret = _fn(hpp_list, tok, env); \ - if (ret == -EINVAL) { \ - pr_err("Invalid --fields key: `%s'", tok); \ - break; \ - } else if (ret == -ESRCH) { \ - pr_err("Unknown --fields key: `%s'", tok); \ - break; \ - } \ - } \ - } while (0) +static int __hpp_list__parse(struct perf_hpp_list *hpp_list, char *_list, struct perf_env *env, + int (*_fn)(struct perf_hpp_list *hpp_list, char *name, struct perf_env *env)) +{ + char *tmp, *tok; + int ret = 0; + + if (!_list) + return 0; + + for (tok = strtok_r(_list, ", ", &tmp); tok; tok = strtok_r(NULL, ", ", &tmp)) { + ret = _fn(hpp_list, tok, env); + switch (ret) { + case 0: + continue; + case -EINVAL: + pr_err("Invalid --fields key: `%s'", tok); + goto out; + case -ESRCH: + pr_err("Unknown --fields key: `%s'", tok); + goto out; + default: { + char buf[STRERR_BUFSIZE]; + + pr_err("%s for --fields key: `%s'", + str_error_r(-ret, buf, sizeof(buf)), tok); + goto out; + } + } + } +out: + return ret; +} static int hpp_list__parse(struct perf_hpp_list *hpp_list, const char *output_, @@ -2093,8 +2107,18 @@ static int hpp_list__parse(struct perf_hpp_list *hpp_list, char *sort = sort_ ? strdup(sort_) : NULL; int ret; - PARSE_LIST(output, c2c_hists__init_output); - PARSE_LIST(sort, c2c_hists__init_sort); + /* strdup() returns NULL on OOM, don't silently treat as empty */ + if ((output_ && !output) || (sort_ && !sort)) { + ret = -ENOMEM; + goto out; + } + + ret = __hpp_list__parse(hpp_list, output, env, c2c_hists__init_output); + if (ret) + goto out; + ret = __hpp_list__parse(hpp_list, sort, env, c2c_hists__init_sort); + if (ret) + goto out; /* copy sort keys to output fields */ perf_hpp__setup_output_field(hpp_list); @@ -2111,6 +2135,7 @@ static int hpp_list__parse(struct perf_hpp_list *hpp_list, perf_hpp__append_sort_keys(&hists->list); #endif +out: free(output); free(sort); return ret; @@ -2121,6 +2146,8 @@ static int c2c_hists__init(struct c2c_hists *hists, int nr_header_lines, struct perf_env *env) { + int ret; + __hists__init(&hists->hists, &hists->list); /* @@ -2133,7 +2160,13 @@ static int c2c_hists__init(struct c2c_hists *hists, /* Overload number of header lines.*/ hists->list.nr_header_lines = nr_header_lines; - return hpp_list__parse(&hists->list, /*output=*/NULL, sort, env); + ret = hpp_list__parse(&hists->list, /*output=*/NULL, sort, env); + + /* Unregister any formats added before the failure point */ + if (ret) + perf_hpp__reset_output_field(&hists->list); + + return ret; } static int c2c_hists__reinit(struct c2c_hists *c2c_hists, @@ -2141,8 +2174,16 @@ static int c2c_hists__reinit(struct c2c_hists *c2c_hists, const char *sort, struct perf_env *env) { + int ret; + perf_hpp__reset_output_field(&c2c_hists->list); - return hpp_list__parse(&c2c_hists->list, output, sort, env); + ret = hpp_list__parse(&c2c_hists->list, output, sort, env); + + /* Unregister any formats added before the failure point */ + if (ret) + perf_hpp__reset_output_field(&c2c_hists->list); + + return ret; } #define DISPLAY_LINE_LIMIT 0.001 @@ -2281,6 +2322,7 @@ static int resort_cl_cb(struct hist_entry *he, void *arg) struct c2c_hist_entry *c2c_he; struct c2c_hists *c2c_hists; bool display = he__display(he, &c2c.shared_clines_stats); + int ret; c2c_he = container_of(he, struct c2c_hist_entry, he); c2c_hists = c2c_he->hists; @@ -2291,7 +2333,9 @@ static int resort_cl_cb(struct hist_entry *he, void *arg) c2c_he->cacheline_idx = idx++; calc_width(c2c_he); - c2c_hists__reinit(c2c_hists, c2c.cl_output, c2c.cl_resort, env); + ret = c2c_hists__reinit(c2c_hists, c2c.cl_output, c2c.cl_resort, env); + if (ret) + return ret; hists__collapse_resort(&c2c_hists->hists, NULL); hists__output_resort_cb(&c2c_hists->hists, NULL, filter_cb); @@ -3356,13 +3400,19 @@ static int perf_c2c__report(int argc, const char **argv) else if (c2c.display == DISPLAY_SNP_PEER) sort_str = "tot_peer"; - c2c_hists__reinit(&c2c.hists, output_str, sort_str, perf_session__env(session)); + err = c2c_hists__reinit(&c2c.hists, output_str, sort_str, perf_session__env(session)); + if (err) { + pr_err("Failed to reinitialize hists\n"); + goto out_mem2node; + } ui_progress__init(&prog, c2c.hists.hists.nr_entries, "Sorting..."); hists__collapse_resort(&c2c.hists.hists, NULL); hists__output_resort_cb(&c2c.hists.hists, &prog, resort_shared_cl_cb); - hists__iterate_cb(&c2c.hists.hists, resort_cl_cb, perf_session__env(session)); + err = hists__iterate_cb(&c2c.hists.hists, resort_cl_cb, perf_session__env(session)); + if (err) + goto out_mem2node; ui_progress__finish(); diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c index 8a7dbfb14535e2..f7126196b09214 100644 --- a/tools/perf/builtin-ftrace.c +++ b/tools/perf/builtin-ftrace.c @@ -72,18 +72,11 @@ static void ftrace__workload_exec_failed_signal(int signo __maybe_unused, static bool check_ftrace_capable(void) { - bool used_root; - - if (perf_cap__capable(CAP_PERFMON, &used_root)) - return true; - - if (!used_root && perf_cap__capable(CAP_SYS_ADMIN, &used_root)) + if (perf_cap__capable(CAP_PERFMON) || + perf_cap__capable(CAP_SYS_ADMIN)) return true; - pr_err("ftrace only works for %s!\n", - used_root ? "root" - : "users with the CAP_PERFMON or CAP_SYS_ADMIN capability" - ); + pr_err("ftrace only works for users with the CAP_PERFMON or CAP_SYS_ADMIN capability!\n"); return false; } @@ -377,9 +370,9 @@ static int set_tracing_pid(struct perf_ftrace *ftrace) if (target__has_cpu(&ftrace->target)) return 0; - for (i = 0; i < perf_thread_map__nr(ftrace->evlist->core.threads); i++) { + for (i = 0; i < perf_thread_map__nr(evlist__core(ftrace->evlist)->threads); i++) { scnprintf(buf, sizeof(buf), "%d", - perf_thread_map__pid(ftrace->evlist->core.threads, i)); + perf_thread_map__pid(evlist__core(ftrace->evlist)->threads, i)); if (append_tracing_file("set_ftrace_pid", buf) < 0) return -1; } @@ -413,7 +406,7 @@ static int set_tracing_cpumask(struct perf_cpu_map *cpumap) static int set_tracing_cpu(struct perf_ftrace *ftrace) { - struct perf_cpu_map *cpumap = ftrace->evlist->core.user_requested_cpus; + struct perf_cpu_map *cpumap = evlist__core(ftrace->evlist)->user_requested_cpus; if (!target__has_cpu(&ftrace->target)) return 0; @@ -1999,20 +1992,20 @@ int cmd_ftrace(int argc, const char **argv) ret = evlist__create_maps(ftrace.evlist, &ftrace.target); if (ret < 0) - goto out_delete_evlist; + goto out_put_evlist; if (argc) { ret = evlist__prepare_workload(ftrace.evlist, &ftrace.target, argv, false, ftrace__workload_exec_failed_signal); if (ret < 0) - goto out_delete_evlist; + goto out_put_evlist; } ret = cmd_func(&ftrace); -out_delete_evlist: - evlist__delete(ftrace.evlist); +out_put_evlist: + evlist__put(ftrace.evlist); out_delete_filters: delete_filter_func(&ftrace.filters); diff --git a/tools/perf/builtin-inject.c b/tools/perf/builtin-inject.c index 6d6cce4765a7f7..b13ce4caf80985 100644 --- a/tools/perf/builtin-inject.c +++ b/tools/perf/builtin-inject.c @@ -1520,7 +1520,7 @@ static int synthesize_id_index(struct perf_inject *inject, size_t new_cnt) struct perf_session *session = inject->session; struct evlist *evlist = session->evlist; struct machine *machine = &session->machines.host; - size_t from = evlist->core.nr_entries - new_cnt; + size_t from = evlist__nr_entries(evlist) - new_cnt; return __perf_event__synthesize_id_index(&inject->tool, perf_event__repipe, evlist, machine, from); @@ -2055,7 +2055,7 @@ static int host__finished_init(const struct perf_tool *tool, struct perf_session if (ret) return ret; - ret = synthesize_id_index(inject, gs->session->evlist->core.nr_entries); + ret = synthesize_id_index(inject, evlist__nr_entries(gs->session->evlist)); if (ret) { pr_err("Failed to synthesize id_index\n"); return ret; diff --git a/tools/perf/builtin-kvm.c b/tools/perf/builtin-kvm.c index 394302ebdb1610..2c6aef1e13a04c 100644 --- a/tools/perf/builtin-kvm.c +++ b/tools/perf/builtin-kvm.c @@ -1221,7 +1221,7 @@ static s64 perf_kvm__mmap_read_idx(struct perf_kvm_stat *kvm, int idx, int err; *mmap_time = ULLONG_MAX; - md = &evlist->mmap[idx]; + md = &evlist__mmap(evlist)[idx]; err = perf_mmap__read_init(&md->core); if (err < 0) return (err == -EAGAIN) ? 0 : -1; @@ -1266,7 +1266,7 @@ static int perf_kvm__mmap_read(struct perf_kvm_stat *kvm) s64 n, ntotal = 0; u64 flush_time = ULLONG_MAX, mmap_time; - for (i = 0; i < kvm->evlist->core.nr_mmaps; i++) { + for (i = 0; i < evlist__core(kvm->evlist)->nr_mmaps; i++) { n = perf_kvm__mmap_read_idx(kvm, i, &mmap_time); if (n < 0) return -1; @@ -1449,7 +1449,7 @@ static int kvm_events_live_report(struct perf_kvm_stat *kvm) evlist__enable(kvm->evlist); while (!done) { - struct fdarray *fda = &kvm->evlist->core.pollfd; + struct fdarray *fda = &evlist__core(kvm->evlist)->pollfd; int rc; rc = perf_kvm__mmap_read(kvm); @@ -1531,7 +1531,7 @@ static int kvm_live_open_events(struct perf_kvm_stat *kvm) goto out; } - if (evlist__mmap(evlist, kvm->opts.mmap_pages) < 0) { + if (evlist__do_mmap(evlist, kvm->opts.mmap_pages) < 0) { ui__error("Failed to mmap the events: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); evlist__close(evlist); @@ -1810,7 +1810,7 @@ static struct evlist *kvm_live_event_list(void) out: if (err) { - evlist__delete(evlist); + evlist__put(evlist); evlist = NULL; } @@ -1931,7 +1931,7 @@ static int kvm_events_live(struct perf_kvm_stat *kvm, perf_session__set_id_hdr_size(kvm->session); ordered_events__set_copy_on_queue(&kvm->session->ordered_events, true); machine__synthesize_threads(&kvm->session->machines.host, &kvm->opts.target, - kvm->evlist->core.threads, true, false, 1); + evlist__core(kvm->evlist)->threads, true, false, 1); err = kvm_live_open_events(kvm); if (err) goto out; @@ -1941,7 +1941,7 @@ static int kvm_events_live(struct perf_kvm_stat *kvm, out: perf_session__delete(kvm->session); kvm->session = NULL; - evlist__delete(kvm->evlist); + evlist__put(kvm->evlist); return err; } diff --git a/tools/perf/builtin-kwork.c b/tools/perf/builtin-kwork.c index 7b61168e01e9d4..fce588441e4996 100644 --- a/tools/perf/builtin-kwork.c +++ b/tools/perf/builtin-kwork.c @@ -1814,7 +1814,7 @@ static int perf_kwork__check_config(struct perf_kwork *kwork, } } - list_for_each_entry(evsel, &session->evlist->core.entries, core.node) { + list_for_each_entry(evsel, &evlist__core(session->evlist)->entries, core.node) { if (kwork->show_callchain && !evsel__has_callchain(evsel)) { pr_debug("Samples do not have callchains\n"); kwork->show_callchain = 0; @@ -1864,9 +1864,9 @@ static int perf_kwork__read_events(struct perf_kwork *kwork) goto out_delete; } - kwork->nr_events = session->evlist->stats.nr_events[0]; - kwork->nr_lost_events = session->evlist->stats.total_lost; - kwork->nr_lost_chunks = session->evlist->stats.nr_events[PERF_RECORD_LOST]; + kwork->nr_events = evlist__stats(session->evlist)->nr_events[0]; + kwork->nr_lost_events = evlist__stats(session->evlist)->total_lost; + kwork->nr_lost_chunks = evlist__stats(session->evlist)->nr_events[PERF_RECORD_LOST]; out_delete: perf_session__delete(session); diff --git a/tools/perf/builtin-lock.c b/tools/perf/builtin-lock.c index 5841d43be97184..d5c0d55cb82d9a 100644 --- a/tools/perf/builtin-lock.c +++ b/tools/perf/builtin-lock.c @@ -2129,7 +2129,7 @@ static int __cmd_contention(int argc, const char **argv) evlist__start_workload(con.evlist); while (!done) { - if (argc && waitpid(con.evlist->workload.pid, NULL, WNOHANG) > 0) + if (argc && waitpid(evlist__workload_pid(con.evlist), NULL, WNOHANG) > 0) break; sleep(1); } @@ -2149,7 +2149,7 @@ static int __cmd_contention(int argc, const char **argv) out_delete: lock_filter_finish(); - evlist__delete(con.evlist); + evlist__put(con.evlist); lock_contention_finish(&con); perf_session__delete(session); perf_env__exit(&host_env); diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c index e915390556752b..a57987851cf03f 100644 --- a/tools/perf/builtin-record.c +++ b/tools/perf/builtin-record.c @@ -64,6 +64,7 @@ #include #include #include +#include #ifndef HAVE_GETTID #include #endif @@ -502,12 +503,12 @@ static void record__aio_mmap_read_sync(struct record *rec) { int i; struct evlist *evlist = rec->evlist; - struct mmap *maps = evlist->mmap; + struct mmap *maps = evlist__mmap(evlist); if (!record__aio_enabled(rec)) return; - for (i = 0; i < evlist->core.nr_mmaps; i++) { + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { struct mmap *map = &maps[i]; if (map->core.base) @@ -653,27 +654,14 @@ static int record__pushfn(struct mmap *map, void *to, void *bf, size_t size) struct record *rec = to; if (record__comp_enabled(rec)) { - struct perf_record_compressed2 *event = map->data; - size_t padding = 0; - u8 pad[8] = {0}; ssize_t compressed = zstd_compress(rec->session, map, map->data, mmap__mmap_len(map), bf, size); if (compressed < 0) return (int)compressed; - bf = event; thread->samples++; - - /* - * The record from `zstd_compress` is not 8 bytes aligned, which would cause asan - * error. We make it aligned here. - */ - event->data_size = compressed - sizeof(struct perf_record_compressed2); - event->header.size = PERF_ALIGN(compressed, sizeof(u64)); - padding = event->header.size - compressed; - return record__write(rec, map, bf, compressed) || - record__write(rec, map, &pad, padding); + return record__write(rec, map, map->data, compressed); } thread->samples++; @@ -811,8 +799,8 @@ static int record__auxtrace_read_snapshot_all(struct record *rec) int i; int rc = 0; - for (i = 0; i < rec->evlist->core.nr_mmaps; i++) { - struct mmap *map = &rec->evlist->mmap[i]; + for (i = 0; i < evlist__core(rec->evlist)->nr_mmaps; i++) { + struct mmap *map = &evlist__mmap(rec->evlist)[i]; if (!map->auxtrace_mmap.base) continue; @@ -1055,15 +1043,15 @@ static void record__thread_data_close_pipes(struct record_thread *thread_data) static bool evlist__per_thread(struct evlist *evlist) { - return cpu_map__is_dummy(evlist->core.user_requested_cpus); + return cpu_map__is_dummy(evlist__core(evlist)->user_requested_cpus); } static int record__thread_data_init_maps(struct record_thread *thread_data, struct evlist *evlist) { - int m, tm, nr_mmaps = evlist->core.nr_mmaps; - struct mmap *mmap = evlist->mmap; - struct mmap *overwrite_mmap = evlist->overwrite_mmap; - struct perf_cpu_map *cpus = evlist->core.all_cpus; + int m, tm, nr_mmaps = evlist__core(evlist)->nr_mmaps; + struct mmap *mmap = evlist__mmap(evlist); + struct mmap *overwrite_mmap = evlist__overwrite_mmap(evlist); + struct perf_cpu_map *cpus = evlist__core(evlist)->all_cpus; bool per_thread = evlist__per_thread(evlist); if (per_thread) @@ -1118,16 +1106,17 @@ static int record__thread_data_init_pollfd(struct record_thread *thread_data, st overwrite_map = thread_data->overwrite_maps ? thread_data->overwrite_maps[tm] : NULL; - for (f = 0; f < evlist->core.pollfd.nr; f++) { - void *ptr = evlist->core.pollfd.priv[f].ptr; + for (f = 0; f < evlist__core(evlist)->pollfd.nr; f++) { + void *ptr = evlist__core(evlist)->pollfd.priv[f].ptr; if ((map && ptr == map) || (overwrite_map && ptr == overwrite_map)) { pos = fdarray__dup_entry_from(&thread_data->pollfd, f, - &evlist->core.pollfd); + &evlist__core(evlist)->pollfd); if (pos < 0) return pos; pr_debug2("thread_data[%p]: pollfd[%d] <- event_fd=%d\n", - thread_data, pos, evlist->core.pollfd.entries[f].fd); + thread_data, pos, + evlist__core(evlist)->pollfd.entries[f].fd); } } } @@ -1171,7 +1160,7 @@ static int record__update_evlist_pollfd_from_thread(struct record *rec, struct evlist *evlist, struct record_thread *thread_data) { - struct pollfd *e_entries = evlist->core.pollfd.entries; + struct pollfd *e_entries = evlist__core(evlist)->pollfd.entries; struct pollfd *t_entries = thread_data->pollfd.entries; int err = 0; size_t i; @@ -1195,7 +1184,7 @@ static int record__dup_non_perf_events(struct record *rec, struct evlist *evlist, struct record_thread *thread_data) { - struct fdarray *fda = &evlist->core.pollfd; + struct fdarray *fda = &evlist__core(evlist)->pollfd; int i, ret; for (i = 0; i < fda->nr; i++) { @@ -1322,17 +1311,17 @@ static int record__mmap_evlist(struct record *rec, return ret; if (record__threads_enabled(rec)) { - ret = perf_data__create_dir(&rec->data, evlist->core.nr_mmaps); + ret = perf_data__create_dir(&rec->data, evlist__core(evlist)->nr_mmaps); if (ret) { errno = -ret; pr_err("Failed to create data directory: %m\n"); return ret; } - for (i = 0; i < evlist->core.nr_mmaps; i++) { - if (evlist->mmap) - evlist->mmap[i].file = &rec->data.dir.files[i]; - if (evlist->overwrite_mmap) - evlist->overwrite_mmap[i].file = &rec->data.dir.files[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + if (evlist__mmap(evlist)) + evlist__mmap(evlist)[i].file = &rec->data.dir.files[i]; + if (evlist__overwrite_mmap(evlist)) + evlist__overwrite_mmap(evlist)[i].file = &rec->data.dir.files[i]; } } @@ -1481,11 +1470,11 @@ out: static void set_timestamp_boundary(struct record *rec, u64 sample_time) { - if (rec->evlist->first_sample_time == 0) - rec->evlist->first_sample_time = sample_time; + if (evlist__first_sample_time(rec->evlist) == 0) + evlist__set_first_sample_time(rec->evlist, sample_time); if (sample_time) - rec->evlist->last_sample_time = sample_time; + evlist__set_last_sample_time(rec->evlist, sample_time); } static int process_sample_event(const struct perf_tool *tool, @@ -1591,18 +1580,36 @@ static void record__adjust_affinity(struct record *rec, struct mmap *map) } } -static size_t process_comp_header(void *record, size_t increment) +/* + * Called once with data_size == 0 to start a record, then once with + * data_size == compressed payload size to finalize and 8-byte-pad it + * (unaligned records trip ASan in the reader). + * Returns the bytes written, or -1 if it won't fit. + */ +static ssize_t process_comp_header(void *record, size_t dst_size, + size_t data_size) { struct perf_record_compressed2 *event = record; size_t size = sizeof(*event); - if (increment) { - event->header.size += increment; - return increment; + if (data_size) { + size_t padding; + + event->data_size = data_size; + event->header.size = PERF_ALIGN(size + data_size, sizeof(u64)); + padding = event->header.size - size - data_size; + if (padding > dst_size) + return -1; + memset(record + size + data_size, 0, padding); + return padding; } + if (size > dst_size) + return -1; + event->header.type = PERF_RECORD_COMPRESSED2; event->header.size = size; + event->data_size = 0; return size; } @@ -1611,7 +1618,12 @@ static ssize_t zstd_compress(struct perf_session *session, struct mmap *map, void *dst, size_t dst_size, void *src, size_t src_size) { ssize_t compressed; - size_t max_record_size = PERF_SAMPLE_MAX_SIZE - sizeof(struct perf_record_compressed2) - 1; + /* + * Reserve space so per-record PERF_ALIGN() padding keeps header.size + * within u16. + */ + size_t max_record_size = PERF_SAMPLE_MAX_SIZE + - sizeof(struct perf_record_compressed2) - sizeof(u64); struct zstd_data *zstd_data = &session->zstd_data; if (map && map->file) @@ -1653,7 +1665,7 @@ static int record__mmap_read_evlist(struct record *rec, struct evlist *evlist, if (!maps) return 0; - if (overwrite && evlist->bkw_mmap_state != BKW_MMAP_DATA_PENDING) + if (overwrite && evlist__bkw_mmap_state(evlist) != BKW_MMAP_DATA_PENDING) return 0; if (record__aio_enabled(rec)) @@ -1808,7 +1820,7 @@ static void record__init_features(struct record *rec) if (rec->no_buildid) perf_header__clear_feat(&session->header, HEADER_BUILD_ID); - if (!have_tracepoints(&rec->evlist->core.entries)) + if (!have_tracepoints(&evlist__core(rec->evlist)->entries)) perf_header__clear_feat(&session->header, HEADER_TRACING_DATA); if (!rec->opts.branch_stack) @@ -1874,7 +1886,7 @@ static int record__synthesize_workload(struct record *rec, bool tail) if (rec->opts.tail_synthesize != tail) return 0; - thread_map = thread_map__new_by_tid(rec->evlist->workload.pid); + thread_map = thread_map__new_by_tid(evlist__workload_pid(rec->evlist)); if (thread_map == NULL) return -1; @@ -2067,10 +2079,10 @@ static void alarm_sig_handler(int sig); static const struct perf_event_mmap_page *evlist__pick_pc(struct evlist *evlist) { if (evlist) { - if (evlist->mmap && evlist->mmap[0].core.base) - return evlist->mmap[0].core.base; - if (evlist->overwrite_mmap && evlist->overwrite_mmap[0].core.base) - return evlist->overwrite_mmap[0].core.base; + if (evlist__mmap(evlist) && evlist__mmap(evlist)[0].core.base) + return evlist__mmap(evlist)[0].core.base; + if (evlist__overwrite_mmap(evlist) && evlist__overwrite_mmap(evlist)[0].core.base) + return evlist__overwrite_mmap(evlist)[0].core.base; } return NULL; } @@ -2150,7 +2162,7 @@ static int record__synthesize(struct record *rec, bool tail) if (err) goto out; - err = perf_event__synthesize_thread_map2(&rec->tool, rec->evlist->core.threads, + err = perf_event__synthesize_thread_map2(&rec->tool, evlist__core(rec->evlist)->threads, process_synthesized_event, NULL); if (err < 0) { @@ -2158,7 +2170,7 @@ static int record__synthesize(struct record *rec, bool tail) return err; } - err = perf_event__synthesize_cpu_map(&rec->tool, rec->evlist->core.all_cpus, + err = perf_event__synthesize_cpu_map(&rec->tool, evlist__core(rec->evlist)->all_cpus, process_synthesized_event, NULL); if (err < 0) { pr_err("Couldn't synthesize cpu map.\n"); @@ -2191,7 +2203,7 @@ static int record__synthesize(struct record *rec, bool tail) bool needs_mmap = rec->opts.synth & PERF_SYNTH_MMAP; err = __machine__synthesize_threads(machine, tool, &opts->target, - rec->evlist->core.threads, + evlist__core(rec->evlist)->threads, f, needs_mmap, opts->record_data_mmap, rec->opts.nr_threads_synthesize); } @@ -2246,6 +2258,8 @@ static int record__setup_sb_evlist(struct record *rec) if (evlist__add_bpf_sb_event(rec->sb_evlist, perf_session__env(rec->session))) { pr_err("Couldn't ask for PERF_RECORD_BPF_EVENT side band events.\n."); + evlist__put(rec->sb_evlist); + rec->sb_evlist = NULL; return -1; } } @@ -2544,7 +2558,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) * because we synthesize event name through the pipe * and need the id for that. */ - if (data->is_pipe && rec->evlist->core.nr_entries == 1) + if (data->is_pipe && evlist__nr_entries(rec->evlist) == 1) rec->opts.sample_id = true; if (rec->timestamp_filename && perf_data__is_pipe(data)) { @@ -2568,7 +2582,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) } /* Debug message used by test scripts */ pr_debug3("perf record done opening and mmapping events\n"); - env->comp_mmap_len = session->evlist->core.mmap_len; + env->comp_mmap_len = evlist__core(session->evlist)->mmap_len; if (rec->opts.kcore) { err = record__kcore_copy(&session->machines.host, data); @@ -2669,7 +2683,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) * Synthesize COMM event to prevent it. */ tgid = perf_event__synthesize_comm(tool, event, - rec->evlist->workload.pid, + evlist__workload_pid(rec->evlist), process_synthesized_event, machine); free(event); @@ -2689,7 +2703,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) * Synthesize NAMESPACES event for the command specified. */ perf_event__synthesize_namespaces(tool, event, - rec->evlist->workload.pid, + evlist__workload_pid(rec->evlist), tgid, process_synthesized_event, machine); free(event); @@ -2706,7 +2720,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) } } - err = event_enable_timer__start(rec->evlist->eet); + err = event_enable_timer__start(evlist__event_enable_timer(rec->evlist)); if (err) goto out_child; @@ -2768,7 +2782,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) * record__mmap_read_all() didn't collect data from * overwritable ring buffer. Read again. */ - if (rec->evlist->bkw_mmap_state == BKW_MMAP_RUNNING) + if (evlist__bkw_mmap_state(rec->evlist) == BKW_MMAP_RUNNING) continue; trigger_ready(&switch_output_trigger); @@ -2837,7 +2851,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) } } - err = event_enable_timer__process(rec->evlist->eet); + err = event_enable_timer__process(evlist__event_enable_timer(rec->evlist)); if (err < 0) goto out_child; if (err) { @@ -2889,11 +2903,13 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) record__synthesize_workload(rec, true); out_child: + evlist__disable(rec->evlist); record__stop_threads(rec); record__mmap_read_all(rec, true); goto out_free_threads; out_child_no_flush: /* mmap read already failed — retrying would just fail again */ + evlist__disable(rec->evlist); record__stop_threads(rec); out_free_threads: record__free_thread_data(rec); @@ -2909,7 +2925,7 @@ out_free_threads: int exit_status; if (!child_finished) - kill(rec->evlist->workload.pid, SIGTERM); + kill(evlist__workload_pid(rec->evlist), SIGTERM); wait(&exit_status); @@ -4032,7 +4048,7 @@ static int record__init_thread_default_masks(struct record *rec, struct perf_cpu static int record__init_thread_masks(struct record *rec) { int ret = 0; - struct perf_cpu_map *cpus = rec->evlist->core.all_cpus; + struct perf_cpu_map *cpus = evlist__core(rec->evlist)->all_cpus; if (!record__threads_enabled(rec)) return record__init_thread_default_masks(rec, cpus); @@ -4283,15 +4299,15 @@ int cmd_record(int argc, const char **argv) if (record.opts.overwrite) record.opts.tail_synthesize = true; - if (rec->evlist->core.nr_entries == 0) { + if (evlist__nr_entries(rec->evlist) == 0) { struct evlist *def_evlist = evlist__new_default(&rec->opts.target, callchain_param.enabled); if (!def_evlist) goto out; - evlist__splice_list_tail(rec->evlist, &def_evlist->core.entries); - evlist__delete(def_evlist); + evlist__splice_list_tail(rec->evlist, &evlist__core(def_evlist)->entries); + evlist__put(def_evlist); } if (rec->opts.target.tid && !rec->opts.no_inherit_set) @@ -4401,7 +4417,7 @@ out: auxtrace_record__free(rec->itr); out_opts: evlist__close_control(rec->opts.ctl_fd, rec->opts.ctl_fd_ack, &rec->opts.ctl_fd_close); - evlist__delete(rec->evlist); + evlist__put(rec->evlist); return err; } diff --git a/tools/perf/builtin-report.c b/tools/perf/builtin-report.c index dd1309c320943e..10db1e5f1e6c42 100644 --- a/tools/perf/builtin-report.c +++ b/tools/perf/builtin-report.c @@ -561,7 +561,7 @@ static int evlist__tty_browse_hists(struct evlist *evlist, struct report *rep, c if (!quiet) { fprintf(stdout, "#\n# Total Lost Samples: %" PRIu64 "\n#\n", - evlist->stats.total_lost_samples); + evlist__stats(evlist)->total_lost_samples); } evlist__for_each_entry(evlist, pos) { @@ -1156,7 +1156,7 @@ static int __cmd_report(struct report *rep) PERF_HPP_REPORT__BLOCK_AVG_CYCLES, }; - if (session->evlist->nr_br_cntr > 0) + if (evlist__nr_br_cntr(session->evlist) > 0) block_hpps[nr_hpps++] = PERF_HPP_REPORT__BLOCK_BRANCH_COUNTER; block_hpps[nr_hpps++] = PERF_HPP_REPORT__BLOCK_RANGE; @@ -1291,7 +1291,7 @@ static int process_attr(const struct perf_tool *tool __maybe_unused, * on events sample_type. */ sample_type = evlist__combined_sample_type(*pevlist); - session = (*pevlist)->session; + session = evlist__session(*pevlist); callchain_param_setup(sample_type, perf_session__e_machine(session, /*e_flags=*/NULL)); return 0; } diff --git a/tools/perf/builtin-sched.c b/tools/perf/builtin-sched.c index 7fd63a9db4574f..d0921bd0ce7582 100644 --- a/tools/perf/builtin-sched.c +++ b/tools/perf/builtin-sched.c @@ -1581,12 +1581,6 @@ static int process_sched_wakeup_event(const struct perf_tool *tool, return 0; } -static int process_sched_wakeup_ignore(const struct perf_tool *tool __maybe_unused, - struct perf_sample *sample __maybe_unused, - struct machine *machine __maybe_unused) -{ - return 0; -} static bool thread__has_color(struct thread *thread) { @@ -1833,7 +1827,7 @@ static int map_switch_event(struct perf_sched *sched, struct perf_sample *sampl sched_out: if (sched->map.task_name) { tr = thread__get_runtime(sched->curr_out_thread[this_cpu.cpu]); - if (strcmp(tr->shortname, "") == 0) + if (tr == NULL || strcmp(tr->shortname, "") == 0) goto out; if (proceed == 1) @@ -1938,6 +1932,22 @@ typedef int (*tracepoint_handler)(const struct perf_tool *tool, struct perf_sample *sample, struct machine *machine); +static struct evsel_str_handler latency_handlers[] = { + { "sched:sched_switch", process_sched_switch_event, }, + { "sched:sched_stat_runtime", process_sched_runtime_event, }, + { "sched:sched_wakeup", process_sched_wakeup_event, }, + { "sched:sched_waking", process_sched_wakeup_event, }, + { "sched:sched_wakeup_new", process_sched_wakeup_event, }, + { "sched:sched_migrate_task", process_sched_migrate_task_event, }, +}; + +static int process_sched_ignore(const struct perf_tool *tool __maybe_unused, + struct perf_sample *sample __maybe_unused, + struct machine *machine __maybe_unused) +{ + return 0; +} + static int perf_sched__process_tracepoint_sample(const struct perf_tool *tool __maybe_unused, union perf_event *event __maybe_unused, struct perf_sample *sample, @@ -1946,7 +1956,23 @@ static int perf_sched__process_tracepoint_sample(const struct perf_tool *tool __ struct evsel *evsel = sample->evsel; int err = 0; - if (evsel->handler != NULL) { + if (evsel->handler == NULL) { + evsel->handler = process_sched_ignore; + for (size_t i = 0; i < ARRAY_SIZE(latency_handlers); i++) { + if (!evsel__name_is(evsel, latency_handlers[i].name)) + continue; + + if (!strcmp(latency_handlers[i].name, "sched:sched_wakeup") && + sample->evsel->evlist && + evlist__find_tracepoint_by_name(sample->evsel->evlist, "sched:sched_waking")) + break; + + evsel->handler = latency_handlers[i].handler; + break; + } + } + + if (evsel->handler != process_sched_ignore) { tracepoint_handler f = evsel->handler; err = f(tool, sample, machine); } @@ -1987,21 +2013,13 @@ static int perf_sched__process_comm(const struct perf_tool *tool __maybe_unused, static int perf_sched__read_events(struct perf_sched *sched) { - struct evsel_str_handler handlers[] = { - { "sched:sched_switch", process_sched_switch_event, }, - { "sched:sched_stat_runtime", process_sched_runtime_event, }, - { "sched:sched_wakeup", process_sched_wakeup_event, }, - { "sched:sched_waking", process_sched_wakeup_event, }, - { "sched:sched_wakeup_new", process_sched_wakeup_event, }, - { "sched:sched_migrate_task", process_sched_migrate_task_event, }, - }; struct perf_session *session; struct perf_data data = { .path = input_name, .mode = PERF_DATA_MODE_READ, .force = sched->force, }; - int rc = -1; + int rc = -1, err; session = perf_session__new(&data, &sched->tool); if (IS_ERR(session)) { @@ -2011,25 +2029,34 @@ static int perf_sched__read_events(struct perf_sched *sched) symbol__init(perf_session__env(session)); - /* prefer sched_waking if it is captured */ - if (evlist__find_tracepoint_by_name(session->evlist, "sched:sched_waking")) - handlers[2].handler = process_sched_wakeup_ignore; + if (!perf_data__is_pipe(session->data)) { + /* prefer sched_waking if it is captured */ + if (evlist__find_tracepoint_by_name(session->evlist, "sched:sched_waking")) + latency_handlers[2].handler = process_sched_ignore; - if (perf_session__set_tracepoints_handlers(session, handlers)) + if (perf_session__set_tracepoints_handlers(session, latency_handlers)) + goto out_delete; + } + + if (!perf_data__is_pipe(session->data) && + !perf_session__has_traces(session, "record -R")) goto out_delete; - if (perf_session__has_traces(session, "record -R")) { - int err = perf_session__process_events(session); - if (err) { - pr_err("Failed to process events, error %d", err); - goto out_delete; - } + err = perf_session__process_events(session); + if (err) { + pr_err("Failed to process events, error %d", err); + goto out_delete; + } - sched->nr_events = session->evlist->stats.nr_events[0]; - sched->nr_lost_events = session->evlist->stats.total_lost; - sched->nr_lost_chunks = session->evlist->stats.nr_events[PERF_RECORD_LOST]; + if (perf_data__is_pipe(session->data) && + !perf_session__has_traces(session, "record -R")) { + goto out_delete; } + sched->nr_events = evlist__stats(session->evlist)->nr_events[0]; + sched->nr_lost_events = evlist__stats(session->evlist)->total_lost; + sched->nr_lost_chunks = evlist__stats(session->evlist)->nr_events[PERF_RECORD_LOST]; + rc = 0; out_delete: perf_session__delete(session); @@ -3303,7 +3330,7 @@ static int timehist_check_attr(struct perf_sched *sched, struct evsel *evsel; struct evsel_runtime *er; - list_for_each_entry(evsel, &evlist->core.entries, core.node) { + list_for_each_entry(evsel, &evlist__core(evlist)->entries, core.node) { er = evsel__get_runtime(evsel); if (er == NULL) { pr_err("Failed to allocate memory for evsel runtime data\n"); @@ -3475,9 +3502,9 @@ static int perf_sched__timehist(struct perf_sched *sched) goto out; } - sched->nr_events = evlist->stats.nr_events[0]; - sched->nr_lost_events = evlist->stats.total_lost; - sched->nr_lost_chunks = evlist->stats.nr_events[PERF_RECORD_LOST]; + sched->nr_events = evlist__stats(evlist)->nr_events[0]; + sched->nr_lost_events = evlist__stats(evlist)->total_lost; + sched->nr_lost_chunks = evlist__stats(evlist)->nr_events[PERF_RECORD_LOST]; if (sched->summary) timehist_print_summary(sched, session); @@ -3924,7 +3951,7 @@ static int perf_sched__schedstat_record(struct perf_sched *sched, session = perf_session__new(&data, &sched->tool); if (IS_ERR(session)) { pr_err("Perf session creation failed.\n"); - evlist__delete(evlist); + evlist__put(evlist); return PTR_ERR(session); } @@ -3982,7 +4009,7 @@ static int perf_sched__schedstat_record(struct perf_sched *sched, if (err < 0) goto out; - user_requested_cpus = evlist->core.user_requested_cpus; + user_requested_cpus = evlist__core(evlist)->user_requested_cpus; err = perf_event__synthesize_schedstat(&(sched->tool), process_synthesized_schedstat_event, @@ -3998,7 +4025,7 @@ static int perf_sched__schedstat_record(struct perf_sched *sched, evlist__start_workload(evlist); while (!done) { - if (argc && waitpid(evlist->workload.pid, NULL, WNOHANG) > 0) + if (argc && waitpid(evlist__workload_pid(evlist), NULL, WNOHANG) > 0) break; sleep(1); } @@ -4023,8 +4050,8 @@ out: else fprintf(stderr, "[ perf sched stats: Failed !! ]\n"); - evlist__delete(evlist); - close(fd); + perf_session__delete(session); + evlist__put(evlist); return err; } @@ -4627,6 +4654,8 @@ static int perf_sched__process_schedstat(const struct perf_tool *tool __maybe_un domain_second_pass = list_first_entry(&cpu_second_pass->domain_head, struct schedstat_domain, domain_list); store_schedstat_cpu_diff(temp); + free(temp->cpu_data); + free(temp); } } else if (event->header.type == PERF_RECORD_SCHEDSTAT_DOMAIN) { struct schedstat_cpu *cpu_tail; @@ -4647,6 +4676,8 @@ static int perf_sched__process_schedstat(const struct perf_tool *tool __maybe_un } else { store_schedstat_domain_diff(temp); domain_second_pass = list_next_entry(domain_second_pass, domain_list); + free(temp->domain_data); + free(temp); } } @@ -4699,7 +4730,7 @@ static int perf_sched__schedstat_report(struct perf_sched *sched) if (err < 0) goto out; - user_requested_cpus = session->evlist->core.user_requested_cpus; + user_requested_cpus = evlist__core(session->evlist)->user_requested_cpus; err = perf_session__process_events(session); @@ -4875,7 +4906,7 @@ static int perf_sched__schedstat_live(struct perf_sched *sched, if (err < 0) goto out; - user_requested_cpus = evlist->core.user_requested_cpus; + user_requested_cpus = evlist__core(evlist)->user_requested_cpus; err = perf_event__synthesize_schedstat(&(sched->tool), process_synthesized_event_live, @@ -4891,7 +4922,7 @@ static int perf_sched__schedstat_live(struct perf_sched *sched, evlist__start_workload(evlist); while (!done) { - if (argc && waitpid(evlist->workload.pid, NULL, WNOHANG) > 0) + if (argc && waitpid(evlist__workload_pid(evlist), NULL, WNOHANG) > 0) break; sleep(1); } @@ -4927,7 +4958,7 @@ static int perf_sched__schedstat_live(struct perf_sched *sched, free_cpu_domain_info(cd_map, sv, nr); out: free_schedstat(&cpu_head); - evlist__delete(evlist); + evlist__put(evlist); return err; } @@ -5167,6 +5198,10 @@ int cmd_sched(int argc, const char **argv) sched.tool.namespaces = perf_event__process_namespaces; sched.tool.lost = perf_event__process_lost; sched.tool.fork = perf_sched__process_fork_event; + sched.tool.attr = perf_event__process_attr; + sched.tool.tracing_data = perf_event__process_tracing_data; + sched.tool.build_id = perf_event__process_build_id; + sched.tool.feature = perf_event__process_feature; argc = parse_options_subcommand(argc, argv, sched_options, sched_subcommands, sched_usage, PARSE_OPT_STOP_AT_NON_OPTION); @@ -5251,19 +5286,20 @@ int cmd_sched(int argc, const char **argv) if (argc) argc = parse_options(argc, argv, stats_options, stats_usage, 0); - return perf_sched__schedstat_record(&sched, argc, argv); + ret = perf_sched__schedstat_record(&sched, argc, argv); } else if (argv[0] && !strcmp(argv[0], "report")) { if (argc) argc = parse_options(argc, argv, stats_options, stats_usage, 0); - return perf_sched__schedstat_report(&sched); + ret = perf_sched__schedstat_report(&sched); } else if (argv[0] && !strcmp(argv[0], "diff")) { if (argc) argc = parse_options(argc, argv, stats_options, stats_usage, 0); - return perf_sched__schedstat_diff(&sched, argc, argv); + ret = perf_sched__schedstat_diff(&sched, argc, argv); + } else { + ret = perf_sched__schedstat_live(&sched, argc, argv); } - return perf_sched__schedstat_live(&sched, argc, argv); } else { usage_with_options(sched_usage, sched_options); } diff --git a/tools/perf/builtin-script.c b/tools/perf/builtin-script.c index 9ac29bdc3cd547..ad8ca08ceb5f26 100644 --- a/tools/perf/builtin-script.c +++ b/tools/perf/builtin-script.c @@ -2174,6 +2174,7 @@ static int script_find_metrics(const struct pmu_metric *pm, struct evsel *metric_evsel; int ret = metricgroup__parse_groups(metric_evlist, /*pmu=*/"all", + /*cputype_filter=*/false, pm->metric_name, /*metric_no_group=*/false, /*metric_no_merge=*/false, @@ -2229,9 +2230,10 @@ static int script_find_metrics(const struct pmu_metric *pm, evlist__for_each_entry(metric_evlist, metric_evsel) { struct evsel *script_evsel = map_metric_evsel_to_script_evsel(script_evlist, metric_evsel); - struct metric_event *metric_me = metricgroup__lookup(&metric_evlist->metric_events, - metric_evsel, - /*create=*/false); + struct metric_event *metric_me = + metricgroup__lookup(evlist__metric_events(metric_evlist), + metric_evsel, + /*create=*/false); if (script_evsel->metric_id == NULL) { script_evsel->metric_id = metric_evsel->metric_id; @@ -2251,7 +2253,7 @@ static int script_find_metrics(const struct pmu_metric *pm, if (metric_me) { struct metric_expr *expr; struct metric_event *script_me = - metricgroup__lookup(&script_evlist->metric_events, + metricgroup__lookup(evlist__metric_events(script_evlist), script_evsel, /*create=*/true); @@ -2274,8 +2276,8 @@ static int script_find_metrics(const struct pmu_metric *pm, } pr_debug("Found metric '%s' whose evsels match those of in the perf data\n", pm->metric_name); - evlist__delete(metric_evlist); out: + evlist__put(metric_evlist); return 0; } @@ -2321,7 +2323,7 @@ static void perf_sample__fprint_metric(struct thread *thread, assert(stat_config.aggr_mode == AGGR_GLOBAL); stat_config.aggr_get_id = script_aggr_cpu_id_get; stat_config.aggr_map = - cpu_aggr_map__new(evsel->evlist->core.user_requested_cpus, + cpu_aggr_map__new(evlist__core(evsel->evlist)->user_requested_cpus, aggr_cpu_id__global, /*data=*/NULL, /*needs_sort=*/false); } @@ -3909,7 +3911,7 @@ static int set_maps(struct perf_script *script) if (WARN_ONCE(script->allocated, "stats double allocation\n")) return -EINVAL; - perf_evlist__set_maps(&evlist->core, script->cpus, script->threads); + perf_evlist__set_maps(evlist__core(evlist), script->cpus, script->threads); if (evlist__alloc_stats(&stat_config, evlist, /*alloc_raw=*/true)) return -ENOMEM; diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c index a04466ea3b0a06..10681a4880086e 100644 --- a/tools/perf/builtin-stat.c +++ b/tools/perf/builtin-stat.c @@ -321,7 +321,7 @@ static int read_single_counter(struct evsel *counter, int cpu_map_idx, int threa */ static int read_counter_cpu(struct evsel *counter, int cpu_map_idx) { - int nthreads = perf_thread_map__nr(evsel_list->core.threads); + int nthreads = perf_thread_map__nr(evlist__core(evsel_list)->threads); int thread; if (!counter->supported) @@ -628,11 +628,12 @@ static int dispatch_events(bool forks, int timeout, int interval, int *times) time_to_sleep = sleep_time; while (!done) { - if (forks) + if (forks) { child_exited = waitpid(child_pid, &status, WNOHANG); - else - child_exited = !is_target_alive(&target, evsel_list->core.threads) ? 1 : 0; - + } else { + child_exited = !is_target_alive(&target, + evlist__core(evsel_list)->threads) ? 1 : 0; + } if (child_exited) break; @@ -681,14 +682,15 @@ static enum counter_recovery stat_handle_error(struct evsel *counter, int err) return COUNTER_RETRY; } if (target__has_per_thread(&target) && err != EOPNOTSUPP && - evsel_list->core.threads && evsel_list->core.threads->err_thread != -1) { + evlist__core(evsel_list)->threads && + evlist__core(evsel_list)->threads->err_thread != -1) { /* * For global --per-thread case, skip current * error thread. */ - if (!thread_map__remove(evsel_list->core.threads, - evsel_list->core.threads->err_thread)) { - evsel_list->core.threads->err_thread = -1; + if (!thread_map__remove(evlist__core(evsel_list)->threads, + evlist__core(evsel_list)->threads->err_thread)) { + evlist__core(evsel_list)->threads->err_thread = -1; counter->supported = true; return COUNTER_RETRY; } @@ -787,11 +789,12 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx) bool second_pass = false, has_supported_counters; if (forks) { - if (evlist__prepare_workload(evsel_list, &target, argv, is_pipe, workload_exec_failed_signal) < 0) { + if (evlist__prepare_workload(evsel_list, &target, argv, is_pipe, + workload_exec_failed_signal) < 0) { perror("failed to prepare workload"); return -1; } - child_pid = evsel_list->workload.pid; + child_pid = evlist__workload_pid(evsel_list); } evlist__for_each_entry(evsel_list, counter) { @@ -1199,7 +1202,7 @@ static int parse_cputype(const struct option *opt, const struct perf_pmu *pmu; struct evlist *evlist = *(struct evlist **)opt->value; - if (!list_empty(&evlist->core.entries)) { + if (!list_empty(&evlist__core(evlist)->entries)) { fprintf(stderr, "Must define cputype before events/metrics\n"); return -1; } @@ -1210,6 +1213,7 @@ static int parse_cputype(const struct option *opt, return -1; } parse_events_option_args.pmu_filter = pmu->name; + parse_events_option_args.cputype_filter = true; return 0; } @@ -1220,12 +1224,13 @@ static int parse_pmu_filter(const struct option *opt, { struct evlist *evlist = *(struct evlist **)opt->value; - if (!list_empty(&evlist->core.entries)) { + if (!list_empty(&evlist__core(evlist)->entries)) { fprintf(stderr, "Must define pmu-filter before events/metrics\n"); return -1; } parse_events_option_args.pmu_filter = str; + parse_events_option_args.cputype_filter = false; return 0; } @@ -1586,8 +1591,9 @@ static int perf_stat_init_aggr_mode(void) if (get_id) { bool needs_sort = stat_config.aggr_mode != AGGR_NONE; - stat_config.aggr_map = cpu_aggr_map__new(evsel_list->core.user_requested_cpus, - get_id, /*data=*/NULL, needs_sort); + stat_config.aggr_map = cpu_aggr_map__new( + evlist__core(evsel_list)->user_requested_cpus, + get_id, /*data=*/NULL, needs_sort); if (!stat_config.aggr_map) { pr_err("cannot build %s map\n", aggr_mode__string[stat_config.aggr_mode]); return -1; @@ -1596,7 +1602,7 @@ static int perf_stat_init_aggr_mode(void) } if (stat_config.aggr_mode == AGGR_THREAD) { - nr = perf_thread_map__nr(evsel_list->core.threads); + nr = perf_thread_map__nr(evlist__core(evsel_list)->threads); stat_config.aggr_map = cpu_aggr_map__empty_new(nr); if (stat_config.aggr_map == NULL) return -ENOMEM; @@ -1615,7 +1621,7 @@ static int perf_stat_init_aggr_mode(void) * taking the highest cpu number to be the size of * the aggregation translate cpumap. */ - nr = perf_cpu_map__max(evsel_list->core.all_cpus).cpu + 1; + nr = perf_cpu_map__max(evlist__core(evsel_list)->all_cpus).cpu + 1; stat_config.cpus_aggr_map = cpu_aggr_map__empty_new(nr); return stat_config.cpus_aggr_map ? 0 : -ENOMEM; } @@ -1902,7 +1908,7 @@ static int perf_stat_init_aggr_mode_file(struct perf_stat *st) bool needs_sort = stat_config.aggr_mode != AGGR_NONE; if (stat_config.aggr_mode == AGGR_THREAD) { - int nr = perf_thread_map__nr(evsel_list->core.threads); + int nr = perf_thread_map__nr(evlist__core(evsel_list)->threads); stat_config.aggr_map = cpu_aggr_map__empty_new(nr); if (stat_config.aggr_map == NULL) @@ -1920,7 +1926,7 @@ static int perf_stat_init_aggr_mode_file(struct perf_stat *st) if (!get_id) return 0; - stat_config.aggr_map = cpu_aggr_map__new(evsel_list->core.user_requested_cpus, + stat_config.aggr_map = cpu_aggr_map__new(evlist__core(evsel_list)->user_requested_cpus, get_id, env, needs_sort); if (!stat_config.aggr_map) { pr_err("cannot build %s map\n", aggr_mode__string[stat_config.aggr_mode]); @@ -1999,7 +2005,9 @@ static int add_default_events(void) ret = -1; goto out; } - ret = metricgroup__parse_groups(evlist, pmu, "transaction", + ret = metricgroup__parse_groups(evlist, pmu, + parse_events_option_args.cputype_filter, + "transaction", stat_config.metric_no_group, stat_config.metric_no_merge, stat_config.metric_no_threshold, @@ -2036,7 +2044,9 @@ static int add_default_events(void) if (!force_metric_only) stat_config.metric_only = true; - ret = metricgroup__parse_groups(evlist, pmu, "smi", + ret = metricgroup__parse_groups(evlist, pmu, + parse_events_option_args.cputype_filter, + "smi", stat_config.metric_no_group, stat_config.metric_no_merge, stat_config.metric_no_threshold, @@ -2073,7 +2083,7 @@ static int add_default_events(void) } str[8] = stat_config.topdown_level + '0'; if (metricgroup__parse_groups(evlist, - pmu, str, + pmu, parse_events_option_args.cputype_filter, str, /*metric_no_group=*/false, /*metric_no_merge=*/false, /*metric_no_threshold=*/true, @@ -2088,7 +2098,7 @@ static int add_default_events(void) if (!stat_config.topdown_level) stat_config.topdown_level = 1; - if (!evlist->core.nr_entries && !evsel_list->core.nr_entries) { + if (!evlist__nr_entries(evlist) && !evlist__nr_entries(evsel_list)) { /* * Add Default metrics. To minimize multiplexing, don't request * threshold computation, but it will be computed if the events @@ -2112,14 +2122,16 @@ static int add_default_events(void) ret = -ENOMEM; break; } - if (metricgroup__parse_groups(metric_evlist, pmu, default_metricgroup_names[i], + if (metricgroup__parse_groups(metric_evlist, pmu, + parse_events_option_args.cputype_filter, + default_metricgroup_names[i], /*metric_no_group=*/false, /*metric_no_merge=*/false, /*metric_no_threshold=*/true, stat_config.user_requested_cpu_list, stat_config.system_wide, stat_config.hardware_aware_grouping) < 0) { - evlist__delete(metric_evlist); + evlist__put(metric_evlist); ret = -1; break; } @@ -2127,13 +2139,13 @@ static int add_default_events(void) evlist__for_each_entry(metric_evlist, evsel) evsel->default_metricgroup = true; - evlist__splice_list_tail(evlist, &metric_evlist->core.entries); + evlist__splice_list_tail(evlist, &evlist__core(metric_evlist)->entries); metricgroup__copy_metric_events(evlist, /*cgrp=*/NULL, - &evlist->metric_events, - &metric_evlist->metric_events); - evlist__delete(metric_evlist); + evlist__metric_events(evlist), + evlist__metric_events(metric_evlist)); + evlist__put(metric_evlist); } - list_sort(/*priv=*/NULL, &evlist->core.entries, default_evlist_evsel_cmp); + list_sort(/*priv=*/NULL, &evlist__core(evlist)->entries, default_evlist_evsel_cmp); } out: @@ -2148,11 +2160,11 @@ out: } } parse_events_error__exit(&err); - evlist__splice_list_tail(evsel_list, &evlist->core.entries); + evlist__splice_list_tail(evsel_list, &evlist__core(evlist)->entries); metricgroup__copy_metric_events(evsel_list, /*cgrp=*/NULL, - &evsel_list->metric_events, - &evlist->metric_events); - evlist__delete(evlist); + evlist__metric_events(evsel_list), + evlist__metric_events(evlist)); + evlist__put(evlist); return ret; } @@ -2272,7 +2284,7 @@ static int set_maps(struct perf_stat *st) if (WARN_ONCE(st->maps_allocated, "stats double allocation\n")) return -EINVAL; - perf_evlist__set_maps(&evsel_list->core, st->cpus, st->threads); + perf_evlist__set_maps(evlist__core(evsel_list), st->cpus, st->threads); if (evlist__alloc_stats(&stat_config, evsel_list, /*alloc_raw=*/true)) return -ENOMEM; @@ -2387,7 +2399,7 @@ static int __cmd_report(int argc, const char **argv) perf_stat.session = session; stat_config.output = stderr; - evlist__delete(evsel_list); + evlist__put(evsel_list); evsel_list = session->evlist; ret = perf_session__process_events(session); @@ -2424,7 +2436,7 @@ static void setup_system_wide(int forks) } } - if (evsel_list->core.nr_entries) + if (evlist__nr_entries(evsel_list)) target.system_wide = true; } } @@ -2651,7 +2663,7 @@ int cmd_stat(int argc, const char **argv) stat_config.csv_sep = DEFAULT_SEPARATOR; if (affinity_set) - evsel_list->no_affinity = !affinity; + evlist__set_no_affinity(evsel_list, !affinity); if (argc && strlen(argv[0]) > 2 && strstarts("record", argv[0])) { argc = __cmd_record(stat_options, &opt_mode, argc, argv); @@ -2714,7 +2726,8 @@ int cmd_stat(int argc, const char **argv) output = fopen(output_name, mode); if (!output) { perror("failed to create output file"); - return -1; + status = -1; + goto out; } if (!stat_config.json_output) { clock_gettime(CLOCK_REALTIME, &tm); @@ -2725,7 +2738,8 @@ int cmd_stat(int argc, const char **argv) output = fdopen(output_fd, mode); if (!output) { perror("Failed opening logfd"); - return -errno; + status = -errno; + goto out; } } @@ -2734,7 +2748,8 @@ int cmd_stat(int argc, const char **argv) parse_options_usage(stat_usage, stat_options, "o", 1); parse_options_usage(NULL, stat_options, "log-fd", 0); parse_options_usage(NULL, stat_options, "interval-clear", 0); - return -1; + status = -1; + goto out; } stat_config.output = output; @@ -2818,7 +2833,7 @@ int cmd_stat(int argc, const char **argv) } if (stat_config.iostat_run) { - status = iostat_prepare(evsel_list, &stat_config); + status = iostat_prepare(&evsel_list, &stat_config); if (status) goto out; if (iostat_mode == IOSTAT_LIST) { @@ -2848,7 +2863,9 @@ int cmd_stat(int argc, const char **argv) */ if (metrics) { const char *pmu = parse_events_option_args.pmu_filter ?: "all"; - int ret = metricgroup__parse_groups(evsel_list, pmu, metrics, + int ret = metricgroup__parse_groups(evsel_list, pmu, + parse_events_option_args.cputype_filter, + metrics, stat_config.metric_no_group, stat_config.metric_no_merge, stat_config.metric_no_threshold, @@ -2882,9 +2899,10 @@ int cmd_stat(int argc, const char **argv) } #ifdef HAVE_BPF_SKEL if (target.use_bpf && nr_cgroups && - (evsel_list->core.nr_entries / nr_cgroups) > BPERF_CGROUP__MAX_EVENTS) { + (evlist__nr_entries(evsel_list) / nr_cgroups) > BPERF_CGROUP__MAX_EVENTS) { pr_warning("Disabling BPF counters due to more events (%d) than the max (%d)\n", - evsel_list->core.nr_entries / nr_cgroups, BPERF_CGROUP__MAX_EVENTS); + evlist__nr_entries(evsel_list) / nr_cgroups, + BPERF_CGROUP__MAX_EVENTS); target.use_bpf = false; } #endif // HAVE_BPF_SKEL @@ -2922,7 +2940,7 @@ int cmd_stat(int argc, const char **argv) * so we could print it out on output. */ if (stat_config.aggr_mode == AGGR_THREAD) { - thread_map__read_comms(evsel_list->core.threads); + thread_map__read_comms(evlist__core(evsel_list)->threads); } if (stat_config.aggr_mode == AGGR_NODE) @@ -3066,7 +3084,7 @@ out: if (smi_cost && smi_reset) sysfs__write_int(FREEZE_ON_SMI_PATH, 0); - evlist__delete(evsel_list); + evlist__put(evsel_list); evlist__close_control(stat_config.ctl_fd, stat_config.ctl_fd_ack, &stat_config.ctl_fd_close); diff --git a/tools/perf/builtin-top.c b/tools/perf/builtin-top.c index 1211401616ee33..570410599f1b37 100644 --- a/tools/perf/builtin-top.c +++ b/tools/perf/builtin-top.c @@ -141,7 +141,7 @@ static int perf_top__parse_source(struct perf_top *top, struct hist_entry *he) notes = symbol__annotation(sym); annotation__lock(notes); - if (!symbol__hists(sym, top->evlist->core.nr_entries)) { + if (!symbol__hists(sym, evlist__nr_entries(top->evlist))) { annotation__unlock(notes); pr_err("Not enough memory for annotating '%s' symbol!\n", sym->name); @@ -267,7 +267,7 @@ static void perf_top__show_details(struct perf_top *top) more = hist_entry__annotate_printf(he, top->sym_evsel); - if (top->evlist->enabled) { + if (evlist__enabled(top->evlist)) { if (top->zero) symbol__annotate_zero_histogram(symbol, top->sym_evsel); else @@ -293,7 +293,7 @@ static void perf_top__resort_hists(struct perf_top *t) */ hists__unlink(hists); - if (evlist->enabled) { + if (evlist__enabled(evlist)) { if (t->zero) { hists__delete_entries(hists); } else { @@ -334,13 +334,13 @@ static void perf_top__print_sym_table(struct perf_top *top) printf("%-*.*s\n", win_width, win_width, graph_dotted_line); if (!top->record_opts.overwrite && - (top->evlist->stats.nr_lost_warned != - top->evlist->stats.nr_events[PERF_RECORD_LOST])) { - top->evlist->stats.nr_lost_warned = - top->evlist->stats.nr_events[PERF_RECORD_LOST]; + (evlist__stats(top->evlist)->nr_lost_warned != + evlist__stats(top->evlist)->nr_events[PERF_RECORD_LOST])) { + evlist__stats(top->evlist)->nr_lost_warned = + evlist__stats(top->evlist)->nr_events[PERF_RECORD_LOST]; color_fprintf(stdout, PERF_COLOR_RED, "WARNING: LOST %d chunks, Check IO/CPU overload", - top->evlist->stats.nr_lost_warned); + evlist__stats(top->evlist)->nr_lost_warned); ++printed; } @@ -447,7 +447,7 @@ static void perf_top__print_mapped_keys(struct perf_top *top) fprintf(stdout, "\t[d] display refresh delay. \t(%d)\n", top->delay_secs); fprintf(stdout, "\t[e] display entries (lines). \t(%d)\n", top->print_entries); - if (top->evlist->core.nr_entries > 1) + if (evlist__nr_entries(top->evlist) > 1) fprintf(stdout, "\t[E] active event counter. \t(%s)\n", evsel__name(top->sym_evsel)); fprintf(stdout, "\t[f] profile display filter (count). \t(%d)\n", top->count_filter); @@ -482,7 +482,7 @@ static int perf_top__key_mapped(struct perf_top *top, int c) case 'S': return 1; case 'E': - return top->evlist->core.nr_entries > 1 ? 1 : 0; + return evlist__nr_entries(top->evlist) > 1 ? 1 : 0; default: break; } @@ -528,7 +528,7 @@ static bool perf_top__handle_keypress(struct perf_top *top, int c) } break; case 'E': - if (top->evlist->core.nr_entries > 1) { + if (evlist__nr_entries(top->evlist) > 1) { /* Select 0 as the default event: */ int counter = 0; @@ -539,7 +539,7 @@ static bool perf_top__handle_keypress(struct perf_top *top, int c) prompt_integer(&counter, "Enter details event counter"); - if (counter >= top->evlist->core.nr_entries) { + if (counter >= evlist__nr_entries(top->evlist)) { top->sym_evsel = evlist__first(top->evlist); fprintf(stderr, "Sorry, no such event, using %s.\n", evsel__name(top->sym_evsel)); sleep(1); @@ -598,8 +598,8 @@ static void perf_top__sort_new_samples(void *arg) { struct perf_top *t = arg; - if (t->evlist->selected != NULL) - t->sym_evsel = t->evlist->selected; + if (evlist__selected(t->evlist) != NULL) + t->sym_evsel = evlist__selected(t->evlist); perf_top__resort_hists(t); @@ -766,7 +766,7 @@ static void perf_event__process_sample(const struct perf_tool *tool, if (!machine) { pr_err("%u unprocessable samples recorded.\r", - top->session->evlist->stats.nr_unprocessable_samples++); + evlist__stats(top->session->evlist)->nr_unprocessable_samples++); return; } @@ -859,7 +859,7 @@ perf_top__process_lost(struct perf_top *top, union perf_event *event, { top->lost += event->lost.lost; top->lost_total += event->lost.lost; - evsel->evlist->stats.total_lost += event->lost.lost; + evlist__stats(evsel->evlist)->total_lost += event->lost.lost; } static void @@ -869,7 +869,7 @@ perf_top__process_lost_samples(struct perf_top *top, { top->lost += event->lost_samples.lost; top->lost_total += event->lost_samples.lost; - evsel->evlist->stats.total_lost_samples += event->lost_samples.lost; + evlist__stats(evsel->evlist)->total_lost_samples += event->lost_samples.lost; } static u64 last_timestamp; @@ -881,7 +881,7 @@ static void perf_top__mmap_read_idx(struct perf_top *top, int idx) struct mmap *md; union perf_event *event; - md = opts->overwrite ? &evlist->overwrite_mmap[idx] : &evlist->mmap[idx]; + md = opts->overwrite ? &evlist__overwrite_mmap(evlist)[idx] : &evlist__mmap(evlist)[idx]; if (perf_mmap__read_init(&md->core) < 0) return; @@ -918,7 +918,7 @@ static void perf_top__mmap_read(struct perf_top *top) if (overwrite) evlist__toggle_bkw_mmap(evlist, BKW_MMAP_DATA_PENDING); - for (i = 0; i < top->evlist->core.nr_mmaps; i++) + for (i = 0; i < evlist__core(top->evlist)->nr_mmaps; i++) perf_top__mmap_read_idx(top, i); if (overwrite) { @@ -1063,7 +1063,7 @@ try_again: goto out_err; } - if (evlist__mmap(evlist, opts->mmap_pages) < 0) { + if (evlist__do_mmap(evlist, opts->mmap_pages) < 0) { ui__error("Failed to mmap with %d (%s)\n", errno, str_error_r(errno, msg, sizeof(msg))); goto out_err; @@ -1218,10 +1218,10 @@ static int deliver_event(struct ordered_events *qe, } else if (event->header.type == PERF_RECORD_LOST_SAMPLES) { perf_top__process_lost_samples(top, event, evsel); } else if (event->header.type < PERF_RECORD_MAX) { - events_stats__inc(&session->evlist->stats, event->header.type); + events_stats__inc(evlist__stats(session->evlist), event->header.type); machine__process_event(machine, event, &sample); } else - ++session->evlist->stats.nr_unknown_events; + ++evlist__stats(session->evlist)->nr_unknown_events; ret = 0; next_event: @@ -1296,7 +1296,7 @@ static int __cmd_top(struct perf_top *top) pr_debug("Couldn't synthesize cgroup events.\n"); machine__synthesize_threads(&top->session->machines.host, &opts->target, - top->evlist->core.threads, true, false, + evlist__core(top->evlist)->threads, true, false, top->nr_threads_synthesize); perf_set_multithreaded(); @@ -1652,14 +1652,14 @@ int cmd_top(int argc, const char **argv) perf_env__init(&host_env); status = perf_config(perf_top_config, &top); if (status) - goto out_delete_evlist; + goto out_put_evlist; /* * Since the per arch annotation init routine may need the cpuid, read * it here, since we are not getting this from the perf.data header. */ status = perf_env__set_cmdline(&host_env, argc, argv); if (status) - goto out_delete_evlist; + goto out_put_evlist; status = perf_env__read_cpuid(&host_env); if (status) { @@ -1680,30 +1680,30 @@ int cmd_top(int argc, const char **argv) annotate_opts.disassembler_style = strdup(disassembler_style); if (!annotate_opts.disassembler_style) { status = -ENOMEM; - goto out_delete_evlist; + goto out_put_evlist; } } if (objdump_path) { annotate_opts.objdump_path = strdup(objdump_path); if (!annotate_opts.objdump_path) { status = -ENOMEM; - goto out_delete_evlist; + goto out_put_evlist; } } if (addr2line_path) { symbol_conf.addr2line_path = strdup(addr2line_path); if (!symbol_conf.addr2line_path) { status = -ENOMEM; - goto out_delete_evlist; + goto out_put_evlist; } } status = symbol__validate_sym_arguments(); if (status) - goto out_delete_evlist; + goto out_put_evlist; if (annotate_check_args() < 0) - goto out_delete_evlist; + goto out_put_evlist; status = target__validate(target); if (status) { @@ -1714,19 +1714,19 @@ int cmd_top(int argc, const char **argv) if (target__none(target)) target->system_wide = true; - if (!top.evlist->core.nr_entries) { + if (!evlist__nr_entries(top.evlist)) { struct evlist *def_evlist = evlist__new_default(target, callchain_param.enabled); if (!def_evlist) - goto out_delete_evlist; + goto out_put_evlist; - evlist__splice_list_tail(top.evlist, &def_evlist->core.entries); - evlist__delete(def_evlist); + evlist__splice_list_tail(top.evlist, &evlist__core(def_evlist)->entries); + evlist__put(def_evlist); } status = evswitch__init(&top.evswitch, top.evlist, stderr); if (status) - goto out_delete_evlist; + goto out_put_evlist; if (symbol_conf.report_hierarchy) { /* disable incompatible options */ @@ -1737,18 +1737,18 @@ int cmd_top(int argc, const char **argv) pr_err("Error: --hierarchy and --fields options cannot be used together\n"); parse_options_usage(top_usage, options, "fields", 0); parse_options_usage(NULL, options, "hierarchy", 0); - goto out_delete_evlist; + goto out_put_evlist; } } if (top.stitch_lbr && !(callchain_param.record_mode == CALLCHAIN_LBR)) { pr_err("Error: --stitch-lbr must be used with --call-graph lbr\n"); - goto out_delete_evlist; + goto out_put_evlist; } if (nr_cgroups > 0 && opts->record_cgroup) { pr_err("--cgroup and --all-cgroups cannot be used together\n"); - goto out_delete_evlist; + goto out_put_evlist; } if (branch_call_mode) { @@ -1772,7 +1772,7 @@ int cmd_top(int argc, const char **argv) status = perf_env__read_core_pmu_caps(&host_env); if (status) { pr_err("PMU capability data is not available\n"); - goto out_delete_evlist; + goto out_put_evlist; } } @@ -1795,9 +1795,9 @@ int cmd_top(int argc, const char **argv) if (IS_ERR(top.session)) { status = PTR_ERR(top.session); top.session = NULL; - goto out_delete_evlist; + goto out_put_evlist; } - top.evlist->session = top.session; + evlist__set_session(top.evlist, top.session); if (setup_sorting(top.evlist, perf_session__env(top.session)) < 0) { if (sort_order) @@ -1805,7 +1805,7 @@ int cmd_top(int argc, const char **argv) if (field_order) parse_options_usage(sort_order ? NULL : top_usage, options, "fields", 0); - goto out_delete_evlist; + goto out_put_evlist; } if (top.uid_str) { @@ -1814,18 +1814,18 @@ int cmd_top(int argc, const char **argv) if (uid == UINT_MAX) { ui__error("Invalid User: %s", top.uid_str); status = -EINVAL; - goto out_delete_evlist; + goto out_put_evlist; } status = parse_uid_filter(top.evlist, uid); if (status) - goto out_delete_evlist; + goto out_put_evlist; } if (evlist__create_maps(top.evlist, target) < 0) { ui__error("Couldn't create thread/CPU maps: %s\n", errno == ENOENT ? "No such process" : str_error_r(errno, errbuf, sizeof(errbuf))); status = -errno; - goto out_delete_evlist; + goto out_put_evlist; } if (top.delay_secs < 1) @@ -1833,7 +1833,7 @@ int cmd_top(int argc, const char **argv) if (record_opts__config(opts)) { status = -EINVAL; - goto out_delete_evlist; + goto out_put_evlist; } top.sym_evsel = evlist__first(top.evlist); @@ -1848,14 +1848,14 @@ int cmd_top(int argc, const char **argv) status = symbol__annotation_init(); if (status < 0) - goto out_delete_evlist; + goto out_put_evlist; annotation_config__init(); symbol_conf.try_vmlinux_path = (symbol_conf.vmlinux_name == NULL); status = symbol__init(NULL); if (status < 0) - goto out_delete_evlist; + goto out_put_evlist; sort__setup_elide(stdout); @@ -1875,13 +1875,15 @@ int cmd_top(int argc, const char **argv) if (top.sb_evlist == NULL) { pr_err("Couldn't create side band evlist.\n."); status = -EINVAL; - goto out_delete_evlist; + goto out_put_evlist; } if (evlist__add_bpf_sb_event(top.sb_evlist, &host_env)) { pr_err("Couldn't ask for PERF_RECORD_BPF_EVENT side band events.\n."); status = -EINVAL; - goto out_delete_evlist; + evlist__put(top.sb_evlist); + top.sb_evlist = NULL; + goto out_put_evlist; } } #endif @@ -1896,8 +1898,8 @@ int cmd_top(int argc, const char **argv) if (!opts->no_bpf_event) evlist__stop_sb_thread(top.sb_evlist); -out_delete_evlist: - evlist__delete(top.evlist); +out_put_evlist: + evlist__put(top.evlist); perf_session__delete(top.session); annotation_options__exit(); perf_env__exit(&host_env); diff --git a/tools/perf/builtin-trace.c b/tools/perf/builtin-trace.c index ba0f8749fc7d79..0418808dbc4d2f 100644 --- a/tools/perf/builtin-trace.c +++ b/tools/perf/builtin-trace.c @@ -460,10 +460,10 @@ static int evsel__init_tp_ptr_field(struct evsel *evsel, struct tp_field *field, ({ struct syscall_tp *sc = __evsel__syscall_tp(evsel);\ evsel__init_tp_ptr_field(evsel, &sc->name, #name); }) -static void evsel__delete_priv(struct evsel *evsel) +static void evsel__put_and_free_priv(struct evsel *evsel) { zfree(&evsel->priv); - evsel__delete(evsel); + evsel__put(evsel); } static int evsel__init_syscall_tp(struct evsel *evsel) @@ -543,7 +543,7 @@ static struct evsel *perf_evsel__raw_syscall_newtp(const char *direction, void * return evsel; out_delete: - evsel__delete_priv(evsel); + evsel__put_and_free_priv(evsel); return NULL; } @@ -2023,7 +2023,7 @@ static int trace__symbols_init(struct trace *trace, int argc, const char **argv, goto out; err = __machine__synthesize_threads(trace->host, &trace->tool, &trace->opts.target, - evlist->core.threads, trace__tool_process, + evlist__core(evlist)->threads, trace__tool_process, /*needs_mmap=*/callchain_param.enabled && !trace->summary_only, /*mmap_data=*/false, @@ -3207,6 +3207,22 @@ static void bpf_output__fprintf(struct trace *trace, ++trace->nr_events_printed; } +static unsigned char bitmap_byte(const unsigned long *mask, int byte_idx) +{ + unsigned char b_val = 0; + int bit_in_byte; + + for (bit_in_byte = 0; bit_in_byte < 8; bit_in_byte++) { + int b_idx = byte_idx * 8 + bit_in_byte; + int host_w_idx = b_idx / BITS_PER_LONG; + int host_bit_in_word = b_idx % BITS_PER_LONG; + + if (mask[host_w_idx] & (1UL << host_bit_in_word)) + b_val |= (1 << bit_in_byte); + } + return b_val; +} + static size_t trace__fprintf_tp_fields(struct trace *trace, struct perf_sample *sample, struct thread *thread, void *augmented_args, int augmented_args_size) { @@ -3238,17 +3254,54 @@ static size_t trace__fprintf_tp_fields(struct trace *trace, struct perf_sample * syscall_arg.len = 0; syscall_arg.fmt = arg; if (field->flags & TEP_FIELD_IS_ARRAY) { - int offset = field->offset; - - if (field->flags & TEP_FIELD_IS_DYNAMIC) { - offset = format_field__intval(field, sample, evsel->needs_swap); - syscall_arg.len = offset >> 16; - offset &= 0xffff; - if (tep_field_is_relative(field->flags)) - offset += field->offset + field->size; + void *ptr = format_field__get_raw_data(field, sample, + evsel->needs_swap, + &syscall_arg.len); + + if (!ptr) { + pr_err("Problem processing %s field, skipping...\n", field->name); + continue; + } + val = (uintptr_t)ptr; + } else if ((field->flags & TEP_FIELD_IS_DYNAMIC) && + strstr(field->type, "cpumask")) { + unsigned long *mask = format_field__get_cpumask(field, sample, + evsel->needs_swap, + &syscall_arg.len); + + if (!mask) { + pr_err("Problem processing %s field, skipping...\n", field->name); + continue; } - val = (uintptr_t)(sample->raw_data + offset); + printed += scnprintf(bf + printed, size - printed, "%s", printed ? ", " : ""); + if (trace->show_arg_names) + printed += scnprintf(bf + printed, size - printed, "%s: ", field->name); + + if (syscall_arg.len == 0) { + printed += scnprintf(bf + printed, size - printed, "0"); + } else { + int i; + bool skip_zero = true; + + printed += scnprintf(bf + printed, size - printed, "0x"); + /* Print bytes from most significant to least significant */ + for (i = syscall_arg.len - 1; i >= 0; i--) { + unsigned char b_val = bitmap_byte(mask, i); + + if (skip_zero && b_val == 0 && i > 0) + continue; + + if (skip_zero) { + printed += scnprintf(bf + printed, size - printed, "%x", b_val); + skip_zero = false; + } else { + printed += scnprintf(bf + printed, size - printed, "%02x", b_val); + } + } + } + free(mask); + continue; } else val = format_field__intval(field, sample, evsel->needs_swap); /* @@ -3633,7 +3686,7 @@ static bool evlist__add_vfs_getname(struct evlist *evlist) list_del_init(&evsel->core.node); evsel->evlist = NULL; - evsel__delete(evsel); + evsel__put(evsel); } return found; @@ -3749,9 +3802,9 @@ out: return ret; out_delete_sys_exit: - evsel__delete_priv(sys_exit); + evsel__put_and_free_priv(sys_exit); out_delete_sys_enter: - evsel__delete_priv(sys_enter); + evsel__put_and_free_priv(sys_enter); goto out; } @@ -4216,7 +4269,7 @@ static int trace__set_filter_pids(struct trace *trace) err = augmented_syscalls__set_filter_pids(trace->filter_pids.nr, trace->filter_pids.entries); } - } else if (perf_thread_map__pid(trace->evlist->core.threads, 0) == -1) { + } else if (perf_thread_map__pid(evlist__core(trace->evlist)->threads, 0) == -1) { err = trace__set_filter_loop_pids(trace); } @@ -4439,7 +4492,7 @@ static int trace__run(struct trace *trace, int argc, const char **argv) if (trace->summary_bpf) { if (trace_prepare_bpf_summary(trace->summary_mode) < 0) - goto out_delete_evlist; + goto out_put_evlist; if (trace->summary_only) goto create_maps; @@ -4507,19 +4560,19 @@ create_maps: err = evlist__create_maps(evlist, &trace->opts.target); if (err < 0) { fprintf(trace->output, "Problems parsing the target to trace, check your options!\n"); - goto out_delete_evlist; + goto out_put_evlist; } err = trace__symbols_init(trace, argc, argv, evlist); if (err < 0) { fprintf(trace->output, "Problems initializing symbol libraries!\n"); - goto out_delete_evlist; + goto out_put_evlist; } if (trace->summary_mode == SUMMARY__BY_TOTAL && !trace->summary_bpf) { trace->syscall_stats = alloc_syscall_stats(); if (!trace->syscall_stats) - goto out_delete_evlist; + goto out_put_evlist; } evlist__config(evlist, &trace->opts, &callchain_param); @@ -4528,9 +4581,9 @@ create_maps: err = evlist__prepare_workload(evlist, &trace->opts.target, argv, false, NULL); if (err < 0) { fprintf(trace->output, "Couldn't run the workload!\n"); - goto out_delete_evlist; + goto out_put_evlist; } - workload_pid = evlist->workload.pid; + workload_pid = evlist__workload_pid(evlist); } err = evlist__open(evlist); @@ -4576,13 +4629,13 @@ create_maps: err = trace__expand_filters(trace, &evsel); if (err) - goto out_delete_evlist; + goto out_put_evlist; err = evlist__apply_filters(evlist, &evsel, &trace->opts.target); if (err < 0) goto out_error_apply_filters; if (!trace->summary_only || !trace->summary_bpf) { - err = evlist__mmap(evlist, trace->opts.mmap_pages); + err = evlist__do_mmap(evlist, trace->opts.mmap_pages); if (err < 0) goto out_error_mmap; } @@ -4601,8 +4654,8 @@ create_maps: if (trace->summary_bpf) trace_start_bpf_summary(); - trace->multiple_threads = perf_thread_map__pid(evlist->core.threads, 0) == -1 || - perf_thread_map__nr(evlist->core.threads) > 1 || + trace->multiple_threads = perf_thread_map__pid(evlist__core(evlist)->threads, 0) == -1 || + perf_thread_map__nr(evlist__core(evlist)->threads) > 1 || evlist__first(evlist)->core.attr.inherit; /* @@ -4619,11 +4672,11 @@ create_maps: again: before = trace->nr_events; - for (i = 0; i < evlist->core.nr_mmaps; i++) { + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { union perf_event *event; struct mmap *md; - md = &evlist->mmap[i]; + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; @@ -4693,12 +4746,12 @@ out_disable: } } -out_delete_evlist: +out_put_evlist: trace_cleanup_bpf_summary(); delete_syscall_stats(trace->syscall_stats); trace__symbols__exit(trace); evlist__free_syscall_tp_fields(evlist); - evlist__delete(evlist); + evlist__put(evlist); cgroup__put(trace->cgroup); trace->evlist = NULL; trace->live = false; @@ -4723,21 +4776,21 @@ out_error_open: out_error: fprintf(trace->output, "%s\n", errbuf); - goto out_delete_evlist; + goto out_put_evlist; out_error_apply_filters: fprintf(trace->output, "Failed to set filter \"%s\" on event %s: %m\n", evsel->filter, evsel__name(evsel)); - goto out_delete_evlist; + goto out_put_evlist; } out_error_mem: fprintf(trace->output, "Not enough memory to run!\n"); - goto out_delete_evlist; + goto out_put_evlist; out_errno: fprintf(trace->output, "%m\n"); - goto out_delete_evlist; + goto out_put_evlist; } static int trace__replay(struct trace *trace) @@ -5325,7 +5378,7 @@ static int trace__parse_cgroups(const struct option *opt, const char *str, int u { struct trace *trace = opt->value; - if (!list_empty(&trace->evlist->core.entries)) { + if (!list_empty(&evlist__core(trace->evlist)->entries)) { struct option o = { .value = &trace->evlist, }; @@ -5417,7 +5470,7 @@ static void trace__exit(struct trace *trace) zfree(&trace->syscalls.table); } zfree(&trace->perfconfig_events); - evlist__delete(trace->evlist); + evlist__put(trace->evlist); trace->evlist = NULL; ordered_events__free(&trace->oe.data); #ifdef HAVE_LIBBPF_SUPPORT @@ -5599,7 +5652,7 @@ int cmd_trace(int argc, const char **argv) * .perfconfig trace.add_events, and filter those out. */ if (!trace.trace_syscalls && !trace.trace_pgfaults && - trace.evlist->core.nr_entries == 0 /* Was --events used? */) { + evlist__nr_entries(trace.evlist) == 0 /* Was --events used? */) { trace.trace_syscalls = true; } /* @@ -5685,7 +5738,7 @@ skip_augmentation: symbol_conf.use_callchain = true; } - if (trace.evlist->core.nr_entries > 0) { + if (evlist__nr_entries(trace.evlist) > 0) { bool use_btf = false; evlist__set_default_evsel_handler(trace.evlist, trace__event_handler); diff --git a/tools/perf/pmu-events/arch/arm64/nvidia/t410/metrics.json b/tools/perf/pmu-events/arch/arm64/nvidia/t410/metrics.json index b825ede03f5441..59474ccb7417f7 100644 --- a/tools/perf/pmu-events/arch/arm64/nvidia/t410/metrics.json +++ b/tools/perf/pmu-events/arch/arm64/nvidia/t410/metrics.json @@ -346,42 +346,42 @@ "MetricExpr": "l1d_demand_misses / l1d_demand_accesses", "BriefDescription": "This metric measures the ratio of L1 D-cache Read accesses missed to the total number of L1 D-cache accesses. This gives an indication of the effectiveness of the L1 D-cache for demand Load or Store traffic.", "ScaleUnit": "1per cache access", - "MetricGroup": "L1I_Prefetcher_Effectiveness" + "MetricGroup": "L1D_Prefetcher_Effectiveness" }, { "MetricName": "l1d_demand_accesses", "MetricExpr": "L1D_CACHE_RW", "BriefDescription": "This metric measures the count of L1 D-cache accesses incurred on Load or Store by the instruction stream of the program.", "ScaleUnit": "1count", - "MetricGroup": "L1I_Prefetcher_Effectiveness" + "MetricGroup": "L1D_Prefetcher_Effectiveness" }, { "MetricName": "l1d_demand_misses", "MetricExpr": "L1D_CACHE_REFILL_RW", "BriefDescription": "This metric measures the count of L1 D-cache misses incurred on a Load or Store by the instruction stream of the program.", "ScaleUnit": "1count", - "MetricGroup": "L1I_Prefetcher_Effectiveness" + "MetricGroup": "L1D_Prefetcher_Effectiveness" }, { "MetricName": "l1d_prf_accuracy", "MetricExpr": "100 * (l1d_useful_prf / l1d_refilled_prf)", "BriefDescription": "This metric measures the fraction of prefetched memory addresses that are used by the instruction stream.", "ScaleUnit": "1percent of prefetch", - "MetricGroup": "L1I_Prefetcher_Effectiveness" + "MetricGroup": "L1D_Prefetcher_Effectiveness" }, { "MetricName": "l1d_prf_coverage", "MetricExpr": "100 * (l1d_useful_prf / (l1d_demand_misses + l1d_refilled_prf))", "BriefDescription": "This metric measures the baseline demand cache misses which the prefetcher brings into the cache.", "ScaleUnit": "1percent of cache access", - "MetricGroup": "L1I_Prefetcher_Effectiveness" + "MetricGroup": "L1D_Prefetcher_Effectiveness" }, { "MetricName": "l1d_refilled_prf", "MetricExpr": "L1D_CACHE_REFILL_HWPRF + L1D_CACHE_REFILL_PRFM + L1D_LFB_HIT_RW_FHWPRF + L1D_LFB_HIT_RW_FPRFM", "BriefDescription": "This metric measures the count of cache lines refilled by L1 data prefetcher (hardware prefetches or software preload) into L1 D-cache.", "ScaleUnit": "1count", - "MetricGroup": "L1I_Prefetcher_Effectiveness" + "MetricGroup": "L1D_Prefetcher_Effectiveness" }, { "MetricName": "l1d_tlb_miss_ratio", @@ -402,7 +402,7 @@ "MetricExpr": "L1D_CACHE_HIT_RW_FPRF + L1D_LFB_HIT_RW_FHWPRF + L1D_LFB_HIT_RW_FPRFM", "BriefDescription": "This metric measures the count of cache lines refilled by L1 data prefetcher (hardware prefetches or software preload) into L1 D-cache which are further used by Load or Store from the instruction stream of the program.", "ScaleUnit": "1count", - "MetricGroup": "L1I_Prefetcher_Effectiveness" + "MetricGroup": "L1D_Prefetcher_Effectiveness" }, { "MetricName": "l1i_cache_miss_ratio", @@ -423,42 +423,42 @@ "MetricExpr": "l1i_demand_misses / l1i_demand_accesses", "BriefDescription": "This metric measures the ratio of L1 I-cache Read accesses missed to the total number of L1 I-cache accesses. This gives an indication of the effectiveness of the L1 I-cache for demand instruction fetch traffic. Note that cache accesses in this cache are demand instruction fetch.", "ScaleUnit": "1per cache access", - "MetricGroup": "L1D_Prefetcher_Effectiveness" + "MetricGroup": "L1I_Prefetcher_Effectiveness" }, { "MetricName": "l1i_demand_accesses", "MetricExpr": "L1I_CACHE_RD", "BriefDescription": "This metric measures the count of L1 I-cache accesses caused by an instruction fetch by the instruction stream of the program.", "ScaleUnit": "1count", - "MetricGroup": "L1D_Prefetcher_Effectiveness" + "MetricGroup": "L1I_Prefetcher_Effectiveness" }, { "MetricName": "l1i_demand_misses", "MetricExpr": "L1I_CACHE_REFILL_RD", "BriefDescription": "This metric measures the count of L1 I-cache misses caused by an instruction fetch by the instruction stream of the program.", "ScaleUnit": "1count", - "MetricGroup": "L1D_Prefetcher_Effectiveness" + "MetricGroup": "L1I_Prefetcher_Effectiveness" }, { "MetricName": "l1i_prf_accuracy", "MetricExpr": "100 * (l1i_useful_prf / l1i_refilled_prf)", "BriefDescription": "This metric measures the fraction of prefetched memory addresses that are used by the instruction stream.", "ScaleUnit": "1percent of prefetch", - "MetricGroup": "L1D_Prefetcher_Effectiveness" + "MetricGroup": "L1I_Prefetcher_Effectiveness" }, { "MetricName": "l1i_prf_coverage", "MetricExpr": "100 * (l1i_useful_prf / (l1i_demand_misses + l1i_refilled_prf))", "BriefDescription": "This metric measures the baseline demand cache misses which the prefetcher brings into the cache.", "ScaleUnit": "1percent of cache access", - "MetricGroup": "L1D_Prefetcher_Effectiveness" + "MetricGroup": "L1I_Prefetcher_Effectiveness" }, { "MetricName": "l1i_refilled_prf", "MetricExpr": "L1I_CACHE_REFILL_HWPRF + L1I_CACHE_REFILL_PRFM", "BriefDescription": "This metric measures the count of cache lines refilled by L1 instruction prefetcher (hardware prefetches or software preload) into L1 I-cache.", "ScaleUnit": "1count", - "MetricGroup": "L1D_Prefetcher_Effectiveness" + "MetricGroup": "L1I_Prefetcher_Effectiveness" }, { "MetricName": "l1i_tlb_miss_ratio", @@ -479,7 +479,7 @@ "MetricExpr": "L1I_CACHE_HIT_RD_FPRF", "BriefDescription": "This metric measures the count of cache lines refilled by L1 instruction prefetcher (hardware prefetches or software preload) into L1 I-cache which are further used by instruction stream of the program.", "ScaleUnit": "1count", - "MetricGroup": "L1D_Prefetcher_Effectiveness" + "MetricGroup": "L1I_Prefetcher_Effectiveness" }, { "MetricName": "l2_cache_miss_ratio", diff --git a/tools/perf/pmu-events/arch/arm64/nvidia/t410/misc.json b/tools/perf/pmu-events/arch/arm64/nvidia/t410/misc.json index 8ff87d844e5212..e1b45cc33b1802 100644 --- a/tools/perf/pmu-events/arch/arm64/nvidia/t410/misc.json +++ b/tools/perf/pmu-events/arch/arm64/nvidia/t410/misc.json @@ -316,8 +316,8 @@ }, { "EventCode": "0x0197", - "EventName": "TXREQ_LIMIT_1QUARTER_CYCLES", - "PublicDescription": "Number of cycles in which the dynamic TXREQ limit is between 1/4 of the L2_TQ_SIZE and 1/2 of the L2_TQ_SIZE." + "EventName": "TXREQ_LIMIT_BELOW_HALF_CYCLES", + "PublicDescription": "Number of cycles in which the dynamic TXREQ limit is between 0 and 1/2 of the L2_TQ_SIZE." }, { "EventCode": "0x019d", @@ -517,7 +517,7 @@ { "EventCode": "0x01cc", "EventName": "TXREQ_LIMIT_COUNT_CYCLES", - "PublicDescription": "This event increments by the dynamic TXREQ value, in each cycle.\nThis is a companion event of TXREQ_LIMIT_MAX_CYCLES, TXREQ_LIMIT_3QUARTER_CYCLES, TXREQ_LIMIT_HALF_CYCLES, and TXREQ_LIMIT_1QUARTER_CYCLES." + "PublicDescription": "This event increments by the dynamic TXREQ value, in each cycle.\nThis is a companion event of TXREQ_LIMIT_MAX_CYCLES, TXREQ_LIMIT_3QUARTER_CYCLES, TXREQ_LIMIT_HALF_CYCLES, and TXREQ_LIMIT_BELOW_HALF_CYCLES." }, { "EventCode": "0x01ce", diff --git a/tools/perf/pmu-events/arch/x86/amdzen5/floating-point.json b/tools/perf/pmu-events/arch/x86/amdzen5/floating-point.json index 9204bfb1d69e0c..50d38434f8d3df 100644 --- a/tools/perf/pmu-events/arch/x86/amdzen5/floating-point.json +++ b/tools/perf/pmu-events/arch/x86/amdzen5/floating-point.json @@ -179,6 +179,30 @@ "BriefDescription": "Retired scalar floating-point blend ops.", "UMask": "0x09" }, + { + "EventName": "fp_ops_retired_by_type.scalar_mov", + "EventCode": "0x0a", + "BriefDescription": "Retired scalar floating-point MOV ops.", + "UMask": "0x0a" + }, + { + "EventName": "fp_ops_retired_by_type.scalar_shuffle", + "EventCode": "0x0a", + "BriefDescription": "Retired scalar floating-point shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", + "UMask": "0x0b" + }, + { + "EventName": "fp_ops_retired_by_type.scalar_bfloat", + "EventCode": "0x0a", + "BriefDescription": "Retired scalar floating-point bfloat ops.", + "UMask": "0x0c" + }, + { + "EventName": "fp_ops_retired_by_type.scalar_logical", + "EventCode": "0x0a", + "BriefDescription": "Retired scalar floating-point logical ops.", + "UMask": "0x0d" + }, { "EventName": "fp_ops_retired_by_type.scalar_other", "EventCode": "0x0a", @@ -245,12 +269,24 @@ "BriefDescription": "Retired vector floating-point blend ops.", "UMask": "0x90" }, + { + "EventName": "fp_ops_retired_by_type.vector_mov", + "EventCode": "0x0a", + "BriefDescription": "Retired vector floating-point MOV ops.", + "UMask": "0xa0" + }, { "EventName": "fp_ops_retired_by_type.vector_shuffle", "EventCode": "0x0a", "BriefDescription": "Retired vector floating-point shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0xb0" }, + { + "EventName": "fp_ops_retired_by_type.vector_bfloat", + "EventCode": "0x0a", + "BriefDescription": "Retired vector floating-point bfloat ops.", + "UMask": "0xc0" + }, { "EventName": "fp_ops_retired_by_type.vector_logical", "EventCode": "0x0a", @@ -278,7 +314,7 @@ { "EventName": "sse_avx_ops_retired.mmx_add", "EventCode": "0x0b", - "BriefDescription": "Retired MMX integer add.", + "BriefDescription": "Retired MMX integer add ops.", "UMask": "0x01" }, { @@ -299,16 +335,34 @@ "BriefDescription": "Retired MMX integer multiply-accumulate ops.", "UMask": "0x04" }, + { + "EventName": "sse_avx_ops_retired.mmx_aes", + "EventCode": "0x0b", + "BriefDescription": "Retired MMX integer AES ops.", + "UMask": "0x05" + }, + { + "EventName": "sse_avx_ops_retired.mmx_sha", + "EventCode": "0x0b", + "BriefDescription": "Retired MMX integer SHA ops.", + "UMask": "0x06" + }, { "EventName": "sse_avx_ops_retired.mmx_cmp", "EventCode": "0x0b", "BriefDescription": "Retired MMX integer compare ops.", "UMask": "0x07" }, + { + "EventName": "sse_avx_ops_retired.mmx_cvt", + "EventCode": "0x0b", + "BriefDescription": "Retired MMX integer convert or pack ops.", + "UMask": "0x08" + }, { "EventName": "sse_avx_ops_retired.mmx_shift", "EventCode": "0x0b", - "BriefDescription": "Retired MMX integer shift ops.", + "BriefDescription": "Retired MMX integer shift or rotate ops.", "UMask": "0x09" }, { @@ -323,10 +377,17 @@ "BriefDescription": "Retired MMX integer shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0x0b" }, + { + "EventName": "sse_avx_ops_retired.mmx_vnni", + "EventCode": "0x0b", + "BriefDescription": "Retired MMX integer VNNI ops.", + "UMask": "0x0c" + }, { "EventName": "sse_avx_ops_retired.mmx_pack", "EventCode": "0x0b", - "BriefDescription": "Retired MMX integer pack ops.", + "BriefDescription": "This event is deprecated. Refer to new event sse_avx_ops_retired.mmx_vnni", + "Deprecated": "1", "UMask": "0x0c" }, { @@ -389,16 +450,23 @@ "BriefDescription": "Retired SSE and AVX integer compare ops.", "UMask": "0x70" }, + { + "EventName": "sse_avx_ops_retired.sse_avx_cvt", + "EventCode": "0x0b", + "BriefDescription": "Retired SSE and AVX integer convert or pack ops.", + "UMask": "0x80" + }, { "EventName": "sse_avx_ops_retired.sse_avx_clm", "EventCode": "0x0b", - "BriefDescription": "Retired SSE and AVX integer CLM ops.", + "BriefDescription": "This event is deprecated. Refer to new event sse_avx_ops_retired.sse_avx_cvt", + "Deprecated": "1", "UMask": "0x80" }, { "EventName": "sse_avx_ops_retired.sse_avx_shift", "EventCode": "0x0b", - "BriefDescription": "Retired SSE and AVX integer shift ops.", + "BriefDescription": "Retired SSE and AVX integer shift or rotate ops.", "UMask": "0x90" }, { @@ -413,10 +481,17 @@ "BriefDescription": "Retired SSE and AVX integer shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0xb0" }, + { + "EventName": "sse_avx_ops_retired.sse_avx_vnni", + "EventCode": "0x0b", + "BriefDescription": "Retired SSE and AVX integer VNNI ops.", + "UMask": "0xc0" + }, { "EventName": "sse_avx_ops_retired.sse_avx_pack", "EventCode": "0x0b", - "BriefDescription": "Retired SSE and AVX integer pack ops.", + "BriefDescription": "This event is deprecated. Refer to new event sse_avx_ops_retired.sse_avx_vnni", + "Deprecated": "1", "UMask": "0xc0" }, { @@ -497,12 +572,24 @@ "BriefDescription": "Retired 128-bit packed floating-point blend ops.", "UMask": "0x09" }, + { + "EventName": "fp_pack_ops_retired.fp128_mov", + "EventCode": "0x0c", + "BriefDescription": "Retired 128-bit packed floating-point MOV ops.", + "UMask": "0x0a" + }, { "EventName": "fp_pack_ops_retired.fp128_shuffle", "EventCode": "0x0c", "BriefDescription": "Retired 128-bit packed floating-point shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0x0b" }, + { + "EventName": "fp_pack_ops_retired.fp128_bfloat", + "EventCode": "0x0c", + "BriefDescription": "Retired 128-bit packed floating-point bfloat ops.", + "UMask": "0x0c" + }, { "EventName": "fp_pack_ops_retired.fp128_logical", "EventCode": "0x0c", @@ -575,12 +662,24 @@ "BriefDescription": "Retired 256-bit packed floating-point blend ops.", "UMask": "0x90" }, + { + "EventName": "fp_pack_ops_retired.fp256_mov", + "EventCode": "0x0c", + "BriefDescription": "Retired 256-bit packed floating-point MOV ops.", + "UMask": "0xa0" + }, { "EventName": "fp_pack_ops_retired.fp256_shuffle", "EventCode": "0x0c", "BriefDescription": "Retired 256-bit packed floating-point shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0xb0" }, + { + "EventName": "fp_pack_ops_retired.fp256_bfloat", + "EventCode": "0x0c", + "BriefDescription": "Retired 256-bit packed floating-point bfloat ops.", + "UMask": "0xc0" + }, { "EventName": "fp_pack_ops_retired.fp256_logical", "EventCode": "0x0c", @@ -647,16 +746,23 @@ "BriefDescription": "Retired 128-bit packed integer compare ops.", "UMask": "0x07" }, + { + "EventName": "packed_int_op_type.int128_cvt", + "EventCode": "0x0d", + "BriefDescription": "Retired 128-bit packed integer convert or pack ops.", + "UMask": "0x08" + }, { "EventName": "packed_int_op_type.int128_clm", "EventCode": "0x0d", - "BriefDescription": "Retired 128-bit packed integer CLM ops.", + "BriefDescription": "This event is deprecated. Refer to new event packed_int_op_type.int128_cvt", + "Deprecated": "1", "UMask": "0x08" }, { "EventName": "packed_int_op_type.int128_shift", "EventCode": "0x0d", - "BriefDescription": "Retired 128-bit packed integer shift ops.", + "BriefDescription": "Retired 128-bit packed integer shift or rotate ops.", "UMask": "0x09" }, { @@ -671,10 +777,17 @@ "BriefDescription": "Retired 128-bit packed integer shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0x0b" }, + { + "EventName": "packed_int_op_type.int128_vnni", + "EventCode": "0x0d", + "BriefDescription": "Retired 128-bit packed integer VNNI ops.", + "UMask": "0x0c" + }, { "EventName": "packed_int_op_type.int128_pack", "EventCode": "0x0d", - "BriefDescription": "Retired 128-bit packed integer pack ops.", + "BriefDescription": "This event is deprecated. Refer to new event packed_int_op_type.int128_vnni", + "Deprecated": "1", "UMask": "0x0c" }, { @@ -719,16 +832,34 @@ "BriefDescription": "Retired 256-bit packed integer multiply-accumulate ops.", "UMask": "0x40" }, + { + "EventName": "packed_int_op_type.int256_aes", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer AES ops.", + "UMask": "0x50" + }, + { + "EventName": "packed_int_op_type.int256_sha", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer SHA ops.", + "UMask": "0x60" + }, { "EventName": "packed_int_op_type.int256_cmp", "EventCode": "0x0d", "BriefDescription": "Retired 256-bit packed integer compare ops.", "UMask": "0x70" }, + { + "EventName": "packed_int_op_type.int256_cvt", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer convert or pack ops.", + "UMask": "0x80" + }, { "EventName": "packed_int_op_type.int256_shift", "EventCode": "0x0d", - "BriefDescription": "Retired 256-bit packed integer shift ops.", + "BriefDescription": "Retired 256-bit packed integer shift or rotate ops.", "UMask": "0x90" }, { @@ -743,10 +874,17 @@ "BriefDescription": "Retired 256-bit packed integer shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0xb0" }, + { + "EventName": "packed_int_op_type.int256_vnni", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer VNNI ops.", + "UMask": "0xc0" + }, { "EventName": "packed_int_op_type.int256_pack", "EventCode": "0x0d", - "BriefDescription": "Retired 256-bit packed integer pack ops.", + "BriefDescription": "This event is deprecated. Refer to new event packed_int_op_type.int256_vnni", + "Deprecated": "1", "UMask": "0xc0" }, { diff --git a/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json b/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json index 06bbaea159259f..b1994539ece820 100644 --- a/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json +++ b/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json @@ -8,9 +8,15 @@ { "EventName": "ls_locks.bus_lock", "EventCode": "0x25", - "BriefDescription": "Retired Lock instructions which caused a bus lock.", + "BriefDescription": "Retired lock instructions which caused a bus lock.", "UMask": "0x01" }, + { + "EventName": "ls_locks.all", + "EventCode": "0x25", + "BriefDescription": "Retired lock instructions of all types.", + "UMask": "0x1f" + }, { "EventName": "ls_ret_cl_flush", "EventCode": "0x26", diff --git a/tools/perf/pmu-events/arch/x86/amdzen6/floating-point.json b/tools/perf/pmu-events/arch/x86/amdzen6/floating-point.json index 03cb039434dee0..71b883776f37b2 100644 --- a/tools/perf/pmu-events/arch/x86/amdzen6/floating-point.json +++ b/tools/perf/pmu-events/arch/x86/amdzen6/floating-point.json @@ -212,7 +212,7 @@ { "EventName": "fp_ops_ret_by_type.scalar_logical", "EventCode": "0x0a", - "BriefDescription": "Retired scalar floating-point move uops.", + "BriefDescription": "Retired scalar floating-point logical uops.", "UMask": "0x0d" }, { @@ -665,6 +665,12 @@ "BriefDescription": "Retired 256-bit packed floating-point shuffle uops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0xb0" }, + { + "EventName": "fp_pack_ops_ret.fp256_bfloat", + "EventCode": "0x0c", + "BriefDescription": "Retired 256-bit packed floating-point bfloat uops.", + "UMask": "0xc0" + }, { "EventName": "fp_pack_ops_ret.fp256_logical", "EventCode": "0x0c", @@ -758,7 +764,7 @@ { "EventName": "fp_pack_int_ops_ret.int128_vnni", "EventCode": "0x0d", - "BriefDescription": "Retired 128-bit packed integer VNNI ops.", + "BriefDescription": "Retired 128-bit packed integer VNNI uops.", "UMask": "0x0c" }, { @@ -803,12 +809,30 @@ "BriefDescription": "Retired 256-bit packed integer multiply-accumulate uops.", "UMask": "0x40" }, + { + "EventName": "fp_pack_int_ops_ret.int256_aes", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer AES uops.", + "UMask": "0x50" + }, + { + "EventName": "fp_pack_int_ops_ret.int256_sha", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer SHA uops.", + "UMask": "0x60" + }, { "EventName": "fp_pack_int_ops_ret.int256_cmp", "EventCode": "0x0d", "BriefDescription": "Retired 256-bit packed integer compare uops.", "UMask": "0x70" }, + { + "EventName": "fp_pack_int_ops_ret.int256_cvt", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer convert or pack uops.", + "UMask": "0x80" + }, { "EventName": "fp_pack_int_ops_ret.int256_shift", "EventCode": "0x0d", @@ -1083,19 +1107,19 @@ "EventName": "fp_nsq_read_stalls.fp_prf", "EventCode": "0x13", "BriefDescription": "Cycles when reads of the NSQ and writes to the floating-point or SIMD schedulers are stalled due to insufficient free physical register file (FP-PRF) entries.", - "UMask": "0x0e" + "UMask": "0x02" }, { "EventName": "fp_nsq_read_stalls.k_prf", "EventCode": "0x13", "BriefDescription": "Cycles when reads of the NSQ and writes to the floating-point or SIMD schedulers are stalled due to insufficient free mask physical register file (K-PRF) entries.", - "UMask": "0x0e" + "UMask": "0x04" }, { "EventName": "fp_nsq_read_stalls.fp_sq", "EventCode": "0x13", "BriefDescription": "Cycles when reads of the NSQ and writes to the floating-point or SIMD schedulers are stalled due to insufficient free scheduler entries.", - "UMask": "0x0e" + "UMask": "0x08" }, { "EventName": "fp_nsq_read_stalls.all", diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py index 376dc2d241621e..3c6cfeefbd5dc3 100755 --- a/tools/perf/pmu-events/jevents.py +++ b/tools/perf/pmu-events/jevents.py @@ -570,13 +570,14 @@ static const struct pmu_table_entry {_pending_events_tblname}[] = {{ def print_pending_metrics() -> None: """Optionally close metrics table.""" - def metric_cmp_key(j: JsonEvent) -> Tuple[bool, str, str]: + def metric_cmp_key(j: JsonEvent) -> Tuple[str, str, str, str]: def fix_none(s: Optional[str]) -> str: if s is None: return '' return s - return (j.desc is not None, fix_none(j.pmu), fix_none(j.metric_name)) + return (fix_none(j.pmu), fix_none(j.metric_name), j.metric_expr.ToPerfJson(), + fix_none(j.desc)) global _pending_metrics if not _pending_metrics: diff --git a/tools/perf/pmu-events/metric.py b/tools/perf/pmu-events/metric.py index a91ccb5977f082..11c7162825f4b8 100644 --- a/tools/perf/pmu-events/metric.py +++ b/tools/perf/pmu-events/metric.py @@ -623,7 +623,11 @@ class Metric: def __lt__(self, other): """Sort order.""" - return self.name < other.name + if self.name != other.name: + return self.name < other.name + if not self.expr.Equals(other.expr): + return self.expr.ToPerfJson() < other.expr.ToPerfJson() + return self.description < other.description def AddToMetricGroup(self, group): """Callback used when being added to a MetricGroup.""" diff --git a/tools/perf/scripts/python/arm-cs-trace-disasm.py b/tools/perf/scripts/python/arm-cs-trace-disasm.py index 8f6fa4a007b42f..42579f85868427 100755 --- a/tools/perf/scripts/python/arm-cs-trace-disasm.py +++ b/tools/perf/scripts/python/arm-cs-trace-disasm.py @@ -31,18 +31,19 @@ from perf_trace_context import perf_sample_srccode, perf_config_get # # Output disassembly with objdump and auto detect vmlinux # (when running on same machine.): -# perf script -s scripts/python/arm-cs-trace-disasm.py -d +# perf script --itrace=b -s scripts/python/arm-cs-trace-disasm.py \ +# -- -d # # Output disassembly with llvm-objdump: -# perf script -s scripts/python/arm-cs-trace-disasm.py \ +# perf script --itrace=b -s scripts/python/arm-cs-trace-disasm.py \ # -- -d llvm-objdump-11 -k path/to/vmlinux # # Output accurate disassembly by passing kcore to script: -# perf script -s scripts/python/arm-cs-trace-disasm.py \ +# perf script --itrace=b -s scripts/python/arm-cs-trace-disasm.py \ # -- -d -k perf.data/kcore_dir/kcore # # Output only source line and symbols: -# perf script -s scripts/python/arm-cs-trace-disasm.py +# perf script --itrace=b -s scripts/python/arm-cs-trace-disasm.py def default_objdump(): config = perf_config_get("annotate.objdump") diff --git a/tools/perf/tests/backward-ring-buffer.c b/tools/perf/tests/backward-ring-buffer.c index c5e7999f281775..2735cc26d7eec3 100644 --- a/tools/perf/tests/backward-ring-buffer.c +++ b/tools/perf/tests/backward-ring-buffer.c @@ -34,8 +34,8 @@ static int count_samples(struct evlist *evlist, int *sample_count, { int i; - for (i = 0; i < evlist->core.nr_mmaps; i++) { - struct mmap *map = &evlist->overwrite_mmap[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + struct mmap *map = &evlist__overwrite_mmap(evlist)[i]; union perf_event *event; perf_mmap__read_init(&map->core); @@ -65,7 +65,7 @@ static int do_test(struct evlist *evlist, int mmap_pages, int err; char sbuf[STRERR_BUFSIZE]; - err = evlist__mmap(evlist, mmap_pages); + err = evlist__do_mmap(evlist, mmap_pages); if (err < 0) { pr_debug("evlist__mmap: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); @@ -77,7 +77,7 @@ static int do_test(struct evlist *evlist, int mmap_pages, evlist__disable(evlist); err = count_samples(evlist, sample_count, comm_count); - evlist__munmap(evlist); + evlist__do_munmap(evlist); return err; } @@ -111,7 +111,7 @@ static int test__backward_ring_buffer(struct test_suite *test __maybe_unused, in err = evlist__create_maps(evlist, &opts.target); if (err < 0) { pr_debug("Not enough memory to create thread/cpu maps\n"); - goto out_delete_evlist; + goto out_put_evlist; } parse_events_error__init(&parse_error); @@ -124,7 +124,7 @@ static int test__backward_ring_buffer(struct test_suite *test __maybe_unused, in if (err) { pr_debug("Failed to parse tracepoint event, try use root\n"); ret = TEST_SKIP; - goto out_delete_evlist; + goto out_put_evlist; } evlist__config(evlist, &opts, NULL); @@ -133,19 +133,19 @@ static int test__backward_ring_buffer(struct test_suite *test __maybe_unused, in if (err < 0) { pr_debug("perf_evlist__open: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } ret = TEST_FAIL; err = do_test(evlist, opts.mmap_pages, &sample_count, &comm_count); if (err != TEST_OK) - goto out_delete_evlist; + goto out_put_evlist; if ((sample_count != NR_ITERS) || (comm_count != NR_ITERS)) { pr_err("Unexpected counter: sample_count=%d, comm_count=%d\n", sample_count, comm_count); - goto out_delete_evlist; + goto out_put_evlist; } evlist__close(evlist); @@ -154,16 +154,16 @@ static int test__backward_ring_buffer(struct test_suite *test __maybe_unused, in if (err < 0) { pr_debug("perf_evlist__open: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } err = do_test(evlist, 1, &sample_count, &comm_count); if (err != TEST_OK) - goto out_delete_evlist; + goto out_put_evlist; ret = TEST_OK; -out_delete_evlist: - evlist__delete(evlist); +out_put_evlist: + evlist__put(evlist); return ret; } diff --git a/tools/perf/tests/builtin-test.c b/tools/perf/tests/builtin-test.c index 7e75f590f225e3..96622232743256 100644 --- a/tools/perf/tests/builtin-test.c +++ b/tools/perf/tests/builtin-test.c @@ -10,37 +10,40 @@ #ifdef HAVE_BACKTRACE_SUPPORT #include #endif -#include -#include #include -#include #include -#include +#include + #include -#include +#include "util/term.h" +#include +#include +#include +#include +#include #include #include -#include -#include "util/term.h" +#include +#include +#include + +#include +#include +#include + #include "builtin.h" +#include "color.h" #include "config.h" +#include "debug.h" #include "hist.h" #include "intlist.h" -#include "tests.h" -#include "debug.h" -#include "color.h" -#include -#include #include "string2.h" #include "symbol.h" +#include "tests-scripts.h" +#include "tests.h" #include "util/rlimit.h" #include "util/strbuf.h" -#include -#include -#include -#include - -#include "tests-scripts.h" +#include "util/term.h" static const char *junit_filename; static struct strbuf junit_xml_buf = STRBUF_INIT; @@ -415,73 +418,73 @@ static char *xml_escape(const char *str) return res ? res : strdup(""); } -static const char *format_test_description(const char *desc, int max_desc_width, - char *buf, size_t buf_sz) +static int get_term_width(void) { - int len = strlen(desc); + struct winsize ws; + int cols = 80; + int term_width; /* - * Clamp to buf_sz to prevent GCC format-truncation warnings - * when terminal width is very large. + * If output is redirected to a file or piped, we don't need to wrap + * or truncate at all. Use a massive virtually infinite terminal width + * so descriptions are printed in full. */ - if (max_desc_width >= (int)buf_sz) - max_desc_width = buf_sz - 1; + if (!isatty(fileno(debug_file()))) + return 10000; - if (len > max_desc_width) { - snprintf(buf, buf_sz, "%.*s...", max_desc_width - 3, desc); - return buf; - } - return desc; + get_term_dimensions(&ws); + if (ws.ws_col > 0) + cols = ws.ws_col; + + /* + * Limit description width to fit on a single line. We subtract 35 + * columns of headroom to allocate space for: + * - The suite index prefix: e.g. " 10.100:" (8 characters) plus 1 space separator. + * - The trailing colon (1 character) and space before status (1 character). + * - The longest status results: e.g. "Skip (some metrics failed)" (26 characters) + * or "Running (XX active)" (20 characters). + * + * A minimum description width of 10 is enforced to ensure names are + * legible even on very narrow consoles. + */ + term_width = cols - 35; + if (term_width < 10) + term_width = 10; + + return term_width; +} + +static int get_max_desc_width(int width) +{ + int term_width = get_term_width(); + + return width > term_width ? term_width : width; } static int print_test_result(struct test_suite *t, int curr_suite, int curr_test_case, int result, int width, int running, const char *err_output, double elapsed) { - char desc_buf[256]; - const char *desc = test_description(t, curr_test_case); - struct winsize ws; - int max_desc_area_width; - int target_desc_area_width; - int desc_padding; - - get_term_dimensions(&ws); - /* - * Total terminal columns minus space for status e.g. " Running (12 active)" - * which is 20 chars, plus a margin of 3 chars = 23 chars. - */ - max_desc_area_width = ws.ws_col - 23; - if (max_desc_area_width < 40) - max_desc_area_width = 40; - - /* Standard test has prefix "%3d: " which is 5 chars */ - target_desc_area_width = width + 5; - if (target_desc_area_width > max_desc_area_width) - target_desc_area_width = max_desc_area_width; + int pad_width = get_max_desc_width(width); + int term_width = get_term_width(); if (test_suite__num_test_cases(t) > 1) { char prefix[32]; int len = snprintf(prefix, sizeof(prefix), "%3d.%1d:", curr_suite + 1, curr_test_case + 1); + int pad = len >= 4 ? pad_width + 4 - len : pad_width; + int trunc = len >= 4 ? term_width + 4 - len : term_width; - desc_padding = target_desc_area_width - (len + 1); - if (desc_padding < 20) - desc_padding = 20; - - desc = format_test_description(desc, desc_padding, desc_buf, sizeof(desc_buf)); - pr_info("%s %-*s:", prefix, desc_padding, desc); + pr_info("%s %-*.*s:", prefix, pad, trunc, + test_description(t, curr_test_case)); } else { - desc_padding = target_desc_area_width - 5; - if (desc_padding < 20) - desc_padding = 20; - - desc = format_test_description(desc, desc_padding, desc_buf, sizeof(desc_buf)); - pr_info("%3d: %-*s:", curr_suite + 1, desc_padding, desc); + pr_info("%3d: %-*.*s:", curr_suite + 1, pad_width, term_width, + test_description(t, curr_test_case)); } switch (result) { case TEST_RUNNING: - color_fprintf(stderr, PERF_COLOR_YELLOW, " Running (%d active)\n", running); + color_fprintf(debug_file(), PERF_COLOR_YELLOW, " Running (%d active)\n", running); break; case TEST_OK: if (test_suite__num_test_cases(t) > 1) @@ -495,9 +498,9 @@ static int print_test_result(struct test_suite *t, int curr_suite, int curr_test summary_tests_skipped++; if (reason) - color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (%s)\n", reason); + color_fprintf(debug_file(), PERF_COLOR_YELLOW, " Skip (%s)\n", reason); else - color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip\n"); + color_fprintf(debug_file(), PERF_COLOR_YELLOW, " Skip\n"); } break; case TEST_FAIL: @@ -511,7 +514,7 @@ static int print_test_result(struct test_suite *t, int curr_suite, int curr_test strbuf_addf_safe(&summary_failed_tests_buf, " %3d: %s\n", curr_suite + 1, test_description(t, curr_test_case)); - color_fprintf(stderr, PERF_COLOR_RED, " FAILED!\n"); + color_fprintf(debug_file(), PERF_COLOR_RED, " FAILED!\n"); break; } @@ -747,6 +750,7 @@ static void finish_test(struct child_test **child_tests, int running_test, int c int ret; struct timespec end_time; double elapsed; + width = get_max_desc_width(width); if (child_test == NULL) { /* Test wasn't started. */ @@ -761,7 +765,8 @@ static void finish_test(struct child_test **child_tests, int running_test, int c * sub test names. */ if (test_suite__num_test_cases(t) > 1 && curr_test_case == 0) - pr_info("%3d: %s:\n", curr_suite + 1, test_description(t, -1)); + pr_info("%3d: %-*.*s:\n", curr_suite + 1, width, width, + test_description(t, -1)); /* * Busy loop reading from the child's stdout/stderr that are set to be @@ -969,6 +974,8 @@ static int finish_tests_parallel(struct child_test **child_tests, size_t num_tes int last_suite_printed = -1; sigset_t set, oldset; + width = get_max_desc_width(width); + sigemptyset(&set); sigaddset(&set, SIGINT); sigaddset(&set, SIGTERM); @@ -1037,8 +1044,11 @@ static int finish_tests_parallel(struct child_test **child_tests, size_t num_tes if (next_child) { if (test_suite__num_test_cases(next_child->test) > 1 && last_suite_printed != next_child->suite_num) { - pr_info("%3d: %s:\n", next_child->suite_num + 1, - test_description(next_child->test, -1)); + pr_info("%3d: %-*.*s:\n", + next_child->suite_num + 1, + width, width, + test_description( + next_child->test, -1)); last_suite_printed = next_child->suite_num; } print_test_result(next_child->test, next_child->suite_num, @@ -1101,7 +1111,8 @@ static int finish_tests_parallel(struct child_test **child_tests, size_t num_tes if (test_suite__num_test_cases(child->test) > 1 && last_suite_printed != child->suite_num) { - pr_info("%3d: %s:\n", child->suite_num + 1, + pr_info("%3d: %-*.*s:\n", child->suite_num + 1, + width, width, test_description(child->test, -1)); last_suite_printed = child->suite_num; } @@ -1225,12 +1236,12 @@ static void print_tests_summary(void) pr_info("Passed subtests : %u\n", summary_subtests_passed); pr_info("Skipped tests : %u\n", summary_tests_skipped); if (summary_tests_failed > 0) { - color_fprintf(stderr, PERF_COLOR_RED, "Failed tests : %u\n", + color_fprintf(debug_file(), PERF_COLOR_RED, "Failed tests : %u\n", summary_tests_failed); pr_info("List of failed tests:\n"); pr_info("%s", summary_failed_tests_buf.buf); } else { - color_fprintf(stderr, PERF_COLOR_GREEN, "Failed tests : 0\n"); + color_fprintf(debug_file(), PERF_COLOR_GREEN, "Failed tests : 0\n"); } if (junit_filename) { @@ -1348,9 +1359,13 @@ static int __cmd_test(struct test_suite **suites, int argc, const char *argv[], if (intlist__find(skiplist, curr_suite + 1)) { if (pass == 1) { - pr_info("%3d: %-*s:", curr_suite + 1, width, + int pad_width = get_max_desc_width(width); + int term_width = get_term_width(); + + pr_info("%3d: %-*.*s:", curr_suite + 1, + pad_width, term_width, test_description(*t, -1)); - color_fprintf(stderr, PERF_COLOR_YELLOW, + color_fprintf(debug_file(), PERF_COLOR_YELLOW, " Skip (user override)\n"); summary_tests_skipped++; if (junit_filename) { @@ -1781,9 +1796,14 @@ int cmd_test(int argc, const char **argv) rlimit__bump_memlock(); suites = build_suites(); - if (!suites) - return errno ? -errno : -ENOMEM; + if (!suites) { + int err = errno; + + intlist__delete(skiplist); + return err ? -err : -ENOMEM; + } ret = __cmd_test(suites, argc, argv, skiplist); free(suites); + intlist__delete(skiplist); return ret; } diff --git a/tools/perf/tests/code-reading.c b/tools/perf/tests/code-reading.c index e82ecdc9577785..f0e8ea8754ef5b 100644 --- a/tools/perf/tests/code-reading.c +++ b/tools/perf/tests/code-reading.c @@ -592,8 +592,8 @@ static int process_events(struct machine *machine, struct evlist *evlist, struct mmap *md; int i, ret; - for (i = 0; i < evlist->core.nr_mmaps; i++) { - md = &evlist->mmap[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; @@ -781,7 +781,7 @@ static int do_test_code_reading(bool try_kcore) goto out_put; } - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); str = events[evidx]; pr_debug("Parsing event '%s'\n", str); @@ -809,8 +809,8 @@ static int do_test_code_reading(bool try_kcore) pr_debug("perf_evlist__open() failed!\n%s\n", errbuf); } - perf_evlist__set_maps(&evlist->core, NULL, NULL); - evlist__delete(evlist); + perf_evlist__set_maps(evlist__core(evlist), NULL, NULL); + evlist__put(evlist); evlist = NULL; continue; } @@ -820,7 +820,7 @@ static int do_test_code_reading(bool try_kcore) if (events[evidx] == NULL) goto out_put; - ret = evlist__mmap(evlist, UINT_MAX); + ret = evlist__do_mmap(evlist, UINT_MAX); if (ret < 0) { pr_debug("evlist__mmap failed\n"); goto out_put; @@ -847,7 +847,7 @@ static int do_test_code_reading(bool try_kcore) out_put: thread__put(thread); out_err: - evlist__delete(evlist); + evlist__put(evlist); perf_cpu_map__put(cpus); perf_thread_map__put(threads); machine__delete(machine); diff --git a/tools/perf/tests/event-times.c b/tools/perf/tests/event-times.c index ae3b98bb42cf8c..56dd37ca760ec8 100644 --- a/tools/perf/tests/event-times.c +++ b/tools/perf/tests/event-times.c @@ -50,7 +50,7 @@ static int attach__enable_on_exec(struct evlist *evlist) static int detach__enable_on_exec(struct evlist *evlist) { - waitpid(evlist->workload.pid, NULL, 0); + waitpid(evlist__workload_pid(evlist), NULL, 0); return 0; } @@ -186,7 +186,7 @@ static int test_times(int (attach)(struct evlist *), err = attach(evlist); if (err == TEST_SKIP) { pr_debug(" SKIP : not enough rights\n"); - evlist__delete(evlist); + evlist__put(evlist); return err; } @@ -205,7 +205,7 @@ static int test_times(int (attach)(struct evlist *), count.ena, count.run); out_err: - evlist__delete(evlist); + evlist__put(evlist); return !err ? TEST_OK : TEST_FAIL; } diff --git a/tools/perf/tests/event_update.c b/tools/perf/tests/event_update.c index facc65e29f20c2..220cc0347747d7 100644 --- a/tools/perf/tests/event_update.c +++ b/tools/perf/tests/event_update.c @@ -92,7 +92,7 @@ static int test__event_update(struct test_suite *test __maybe_unused, int subtes TEST_ASSERT_VAL("failed to allocate ids", !perf_evsel__alloc_id(&evsel->core, 1, 1)); - perf_evlist__id_add(&evlist->core, &evsel->core, 0, 0, 123); + perf_evlist__id_add(evlist__core(evlist), &evsel->core, 0, 0, 123); free((char *)evsel->unit); evsel->unit = strdup("KRAVA"); @@ -117,7 +117,7 @@ static int test__event_update(struct test_suite *test __maybe_unused, int subtes TEST_ASSERT_VAL("failed to synthesize attr update cpus", !perf_event__synthesize_event_update_cpus(&tmp.tool, evsel, process_event_cpus)); - evlist__delete(evlist); + evlist__put(evlist); return 0; } diff --git a/tools/perf/tests/evsel-roundtrip-name.c b/tools/perf/tests/evsel-roundtrip-name.c index 1922cac13a2453..6a220634c52f63 100644 --- a/tools/perf/tests/evsel-roundtrip-name.c +++ b/tools/perf/tests/evsel-roundtrip-name.c @@ -33,7 +33,7 @@ static int perf_evsel__roundtrip_cache_name_test(void) if (err) { pr_debug("Failure to parse cache event '%s' possibly as PMUs don't support it", name); - evlist__delete(evlist); + evlist__put(evlist); continue; } evlist__for_each_entry(evlist, evsel) { @@ -42,7 +42,7 @@ static int perf_evsel__roundtrip_cache_name_test(void) ret = TEST_FAIL; } } - evlist__delete(evlist); + evlist__put(evlist); } } } @@ -66,7 +66,7 @@ static int perf_evsel__name_array_test(const char *const names[], int nr_names) if (err) { pr_debug("failed to parse event '%s', err %d\n", names[i], err); - evlist__delete(evlist); + evlist__put(evlist); ret = TEST_FAIL; continue; } @@ -76,7 +76,7 @@ static int perf_evsel__name_array_test(const char *const names[], int nr_names) ret = TEST_FAIL; } } - evlist__delete(evlist); + evlist__put(evlist); } return ret; } diff --git a/tools/perf/tests/evsel-tp-sched.c b/tools/perf/tests/evsel-tp-sched.c index 226196fb967795..9e456f88a13ac1 100644 --- a/tools/perf/tests/evsel-tp-sched.c +++ b/tools/perf/tests/evsel-tp-sched.c @@ -64,7 +64,7 @@ static int test__perf_evsel__tp_sched_test(struct test_suite *test __maybe_unuse if (evsel__test_field(evsel, "next_prio", 4, true)) ret = TEST_FAIL; - evsel__delete(evsel); + evsel__put(evsel); evsel = evsel__newtp("sched", "sched_wakeup"); @@ -85,7 +85,7 @@ static int test__perf_evsel__tp_sched_test(struct test_suite *test __maybe_unuse if (evsel__test_field(evsel, "target_cpu", 4, true)) ret = TEST_FAIL; - evsel__delete(evsel); + evsel__put(evsel); return ret; } diff --git a/tools/perf/tests/expand-cgroup.c b/tools/perf/tests/expand-cgroup.c index dd547f2f77cc83..04d62611766ad6 100644 --- a/tools/perf/tests/expand-cgroup.c +++ b/tools/perf/tests/expand-cgroup.c @@ -28,7 +28,7 @@ static int test_expand_events(struct evlist *evlist) TEST_ASSERT_VAL("evlist is empty", !evlist__empty(evlist)); - nr_events = evlist->core.nr_entries; + nr_events = evlist__nr_entries(evlist); ev_name = calloc(nr_events, sizeof(*ev_name)); if (ev_name == NULL) { pr_debug("memory allocation failure\n"); @@ -54,7 +54,7 @@ static int test_expand_events(struct evlist *evlist) } ret = TEST_FAIL; - if (evlist->core.nr_entries != nr_events * nr_cgrps) { + if (evlist__nr_entries(evlist) != nr_events * nr_cgrps) { pr_debug("event count doesn't match\n"); goto out; } @@ -106,7 +106,7 @@ static int expand_default_events(void) TEST_ASSERT_VAL("failed to get evlist", evlist); ret = test_expand_events(evlist); - evlist__delete(evlist); + evlist__put(evlist); return ret; } @@ -133,7 +133,7 @@ static int expand_group_events(void) ret = test_expand_events(evlist); out: parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); return ret; } @@ -164,7 +164,7 @@ static int expand_libpfm_events(void) ret = test_expand_events(evlist); out: - evlist__delete(evlist); + evlist__put(evlist); return ret; } @@ -179,7 +179,8 @@ static int expand_metric_events(void) TEST_ASSERT_VAL("failed to get evlist", evlist); pme_test = find_core_metrics_table("testarch", "testcpu"); - ret = metricgroup__parse_groups_test(evlist, pme_test, metric_str); + ret = metricgroup__parse_groups_test(evlist, pme_test, metric_str, + /*cputype_filter=*/false); if (ret < 0) { pr_debug("failed to parse '%s' metric\n", metric_str); goto out; @@ -188,7 +189,7 @@ static int expand_metric_events(void) ret = test_expand_events(evlist); out: - evlist__delete(evlist); + evlist__put(evlist); return ret; } diff --git a/tools/perf/tests/hists_cumulate.c b/tools/perf/tests/hists_cumulate.c index 09ee08085b06b1..9356451a172ea2 100644 --- a/tools/perf/tests/hists_cumulate.c +++ b/tools/perf/tests/hists_cumulate.c @@ -744,7 +744,7 @@ static int test__hists_cumulate(struct test_suite *test __maybe_unused, int subt out: /* tear down everything */ - evlist__delete(evlist); + evlist__put(evlist); machines__exit(&machines); put_fake_samples(); diff --git a/tools/perf/tests/hists_filter.c b/tools/perf/tests/hists_filter.c index ac5affb7afff11..f9eaa511487b4a 100644 --- a/tools/perf/tests/hists_filter.c +++ b/tools/perf/tests/hists_filter.c @@ -332,7 +332,7 @@ static int test__hists_filter(struct test_suite *test __maybe_unused, int subtes out: /* tear down everything */ - evlist__delete(evlist); + evlist__put(evlist); reset_output_field(); machines__exit(&machines); put_fake_samples(); diff --git a/tools/perf/tests/hists_link.c b/tools/perf/tests/hists_link.c index e55990163865e6..d88591bcbe508a 100644 --- a/tools/perf/tests/hists_link.c +++ b/tools/perf/tests/hists_link.c @@ -353,7 +353,7 @@ static int test__hists_link(struct test_suite *test __maybe_unused, int subtest out: /* tear down everything */ - evlist__delete(evlist); + evlist__put(evlist); reset_output_field(); machines__exit(&machines); put_fake_samples(); diff --git a/tools/perf/tests/hists_output.c b/tools/perf/tests/hists_output.c index 5e59dba92e8132..f58c8d18fe3309 100644 --- a/tools/perf/tests/hists_output.c +++ b/tools/perf/tests/hists_output.c @@ -631,7 +631,7 @@ static int test__hists_output(struct test_suite *test __maybe_unused, int subtes out: /* tear down everything */ - evlist__delete(evlist); + evlist__put(evlist); machines__exit(&machines); put_fake_samples(); diff --git a/tools/perf/tests/hwmon_pmu.c b/tools/perf/tests/hwmon_pmu.c index 62e0841a6c310a..e26b3fe3fab15e 100644 --- a/tools/perf/tests/hwmon_pmu.c +++ b/tools/perf/tests/hwmon_pmu.c @@ -184,9 +184,10 @@ static int do_test(size_t i, bool with_pmu, bool with_alias) } ret = TEST_OK; - if (with_pmu ? (evlist->core.nr_entries != 1) : (evlist->core.nr_entries < 1)) { + if (with_pmu ? (evlist__nr_entries(evlist) != 1) + : (evlist__nr_entries(evlist) < 1)) { pr_debug("FAILED %s:%d Unexpected number of events for '%s' of %d\n", - __FILE__, __LINE__, str, evlist->core.nr_entries); + __FILE__, __LINE__, str, evlist__nr_entries(evlist)); ret = TEST_FAIL; goto out; } @@ -215,7 +216,7 @@ static int do_test(size_t i, bool with_pmu, bool with_alias) out: parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); return ret; } diff --git a/tools/perf/tests/keep-tracking.c b/tools/perf/tests/keep-tracking.c index 729cc9cc1cb77f..b760041bed3078 100644 --- a/tools/perf/tests/keep-tracking.c +++ b/tools/perf/tests/keep-tracking.c @@ -37,8 +37,8 @@ static int find_comm(struct evlist *evlist, const char *comm) int i, found; found = 0; - for (i = 0; i < evlist->core.nr_mmaps; i++) { - md = &evlist->mmap[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; while ((event = perf_mmap__read_event(&md->core)) != NULL) { @@ -87,7 +87,7 @@ static int test__keep_tracking(struct test_suite *test __maybe_unused, int subte evlist = evlist__new(); CHECK_NOT_NULL__(evlist); - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); CHECK__(parse_event(evlist, "dummy:u")); CHECK__(parse_event(evlist, "cpu-cycles:u")); @@ -106,7 +106,7 @@ static int test__keep_tracking(struct test_suite *test __maybe_unused, int subte goto out_err; } - CHECK__(evlist__mmap(evlist, UINT_MAX)); + CHECK__(evlist__do_mmap(evlist, UINT_MAX)); /* * First, test that a 'comm' event can be found when the event is @@ -153,7 +153,7 @@ static int test__keep_tracking(struct test_suite *test __maybe_unused, int subte out_err: if (evlist) { evlist__disable(evlist); - evlist__delete(evlist); + evlist__put(evlist); } perf_cpu_map__put(cpus); perf_thread_map__put(threads); diff --git a/tools/perf/tests/mmap-basic.c b/tools/perf/tests/mmap-basic.c index a69cd1046e9aa6..5cec7644952c7d 100644 --- a/tools/perf/tests/mmap-basic.c +++ b/tools/perf/tests/mmap-basic.c @@ -81,7 +81,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest goto out_free_cpus; } - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); for (i = 0; i < nsyscalls; ++i) { char name[64]; @@ -94,7 +94,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest /* Permissions failure, flag the failure as a skip. */ err = TEST_SKIP; } - goto out_delete_evlist; + goto out_put_evlist; } evsels[i]->core.attr.wakeup_events = 1; @@ -106,17 +106,17 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest pr_debug("failed to open counter: %s, " "tweak /proc/sys/kernel/perf_event_paranoid?\n", str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } nr_events[i] = 0; expected_nr_events[i] = 1 + rand() % 127; } - if (evlist__mmap(evlist, 128) < 0) { + if (evlist__do_mmap(evlist, 128) < 0) { pr_debug("failed to mmap events: %d (%s)\n", errno, str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } for (i = 0; i < nsyscalls; ++i) @@ -124,7 +124,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest syscalls[i](); } - md = &evlist->mmap[0]; + md = &evlist__mmap(evlist)[0]; if (perf_mmap__read_init(&md->core) < 0) goto out_init; @@ -134,7 +134,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest if (event->header.type != PERF_RECORD_SAMPLE) { pr_debug("unexpected %s event\n", perf_event__name(event->header.type)); - goto out_delete_evlist; + goto out_put_evlist; } perf_sample__init(&sample, /*all=*/false); @@ -142,7 +142,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest if (err) { pr_err("Can't parse sample, err = %d\n", err); perf_sample__exit(&sample); - goto out_delete_evlist; + goto out_put_evlist; } err = -1; @@ -153,7 +153,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest if (evsel == NULL) { pr_debug("event with id %" PRIu64 " doesn't map to an evsel\n", sample.id); - goto out_delete_evlist; + goto out_put_evlist; } nr_events[evsel->core.idx]++; perf_mmap__consume(&md->core); @@ -168,12 +168,12 @@ out_init: expected_nr_events[evsel->core.idx], evsel__name(evsel), nr_events[evsel->core.idx]); err = -1; - goto out_delete_evlist; + goto out_put_evlist; } } -out_delete_evlist: - evlist__delete(evlist); +out_put_evlist: + evlist__put(evlist); out_free_cpus: perf_cpu_map__put(cpus); out_free_threads: diff --git a/tools/perf/tests/openat-syscall-all-cpus.c b/tools/perf/tests/openat-syscall-all-cpus.c index 0be43f8db3bda2..cc63df2b3bc53a 100644 --- a/tools/perf/tests/openat-syscall-all-cpus.c +++ b/tools/perf/tests/openat-syscall-all-cpus.c @@ -59,7 +59,7 @@ static int test__openat_syscall_event_on_all_cpus(struct test_suite *test __mayb "tweak /proc/sys/kernel/perf_event_paranoid?\n", str_error_r(errno, sbuf, sizeof(sbuf))); err = TEST_SKIP; - goto out_evsel_delete; + goto out_evsel_put; } perf_cpu_map__for_each_cpu(cpu, idx, cpus) { @@ -116,8 +116,8 @@ static int test__openat_syscall_event_on_all_cpus(struct test_suite *test __mayb evsel__free_counts(evsel); out_close_fd: perf_evsel__close_fd(&evsel->core); -out_evsel_delete: - evsel__delete(evsel); +out_evsel_put: + evsel__put(evsel); out_cpu_map_delete: perf_cpu_map__put(cpus); out_thread_map_delete: diff --git a/tools/perf/tests/openat-syscall-tp-fields.c b/tools/perf/tests/openat-syscall-tp-fields.c index 9ff8caff98c3a3..5365889d326f8b 100644 --- a/tools/perf/tests/openat-syscall-tp-fields.c +++ b/tools/perf/tests/openat-syscall-tp-fields.c @@ -51,7 +51,7 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused if (IS_ERR(evsel)) { pr_debug("%s: evsel__newtp\n", __func__); ret = PTR_ERR(evsel) == -EACCES ? TEST_SKIP : TEST_FAIL; - goto out_delete_evlist; + goto out_put_evlist; } evlist__add(evlist, evsel); @@ -59,25 +59,25 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused err = evlist__create_maps(evlist, &opts.target); if (err < 0) { pr_debug("%s: evlist__create_maps\n", __func__); - goto out_delete_evlist; + goto out_put_evlist; } evsel__config(evsel, &opts, NULL); - perf_thread_map__set_pid(evlist->core.threads, 0, getpid()); + perf_thread_map__set_pid(evlist__core(evlist)->threads, 0, getpid()); err = evlist__open(evlist); if (err < 0) { pr_debug("perf_evlist__open: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } - err = evlist__mmap(evlist, UINT_MAX); + err = evlist__do_mmap(evlist, UINT_MAX); if (err < 0) { pr_debug("evlist__mmap: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } evlist__enable(evlist); @@ -90,11 +90,11 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused while (1) { int before = nr_events; - for (i = 0; i < evlist->core.nr_mmaps; i++) { + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { union perf_event *event; struct mmap *md; - md = &evlist->mmap[i]; + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; @@ -115,7 +115,7 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused if (err) { pr_debug("Can't parse sample, err = %d\n", err); perf_sample__exit(&sample); - goto out_delete_evlist; + goto out_put_evlist; } tp_flags = perf_sample__intval(&sample, "flags"); @@ -126,7 +126,7 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused (tp_flags & flags) != flags) { pr_debug("%s: Expected flags=%#x, got %#x\n", __func__, flags, tp_flags); - goto out_delete_evlist; + goto out_put_evlist; } goto out_ok; @@ -139,13 +139,13 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused if (++nr_polls > 5) { pr_debug("%s: no events!\n", __func__); - goto out_delete_evlist; + goto out_put_evlist; } } out_ok: ret = TEST_OK; -out_delete_evlist: - evlist__delete(evlist); +out_put_evlist: + evlist__put(evlist); out: return ret; } diff --git a/tools/perf/tests/openat-syscall.c b/tools/perf/tests/openat-syscall.c index b54cbe5f18085e..9f16f0dd3a2959 100644 --- a/tools/perf/tests/openat-syscall.c +++ b/tools/perf/tests/openat-syscall.c @@ -42,7 +42,7 @@ static int test__openat_syscall_event(struct test_suite *test __maybe_unused, "tweak /proc/sys/kernel/perf_event_paranoid?\n", str_error_r(errno, sbuf, sizeof(sbuf))); err = TEST_SKIP; - goto out_evsel_delete; + goto out_evsel_put; } for (i = 0; i < nr_openat_calls; ++i) { @@ -64,8 +64,8 @@ static int test__openat_syscall_event(struct test_suite *test __maybe_unused, err = TEST_OK; out_close_fd: perf_evsel__close_fd(&evsel->core); -out_evsel_delete: - evsel__delete(evsel); +out_evsel_put: + evsel__put(evsel); out_thread_map_delete: perf_thread_map__put(threads); return err; diff --git a/tools/perf/tests/parse-events.c b/tools/perf/tests/parse-events.c index 05c3e899b42517..5f6f95c8a5b40a 100644 --- a/tools/perf/tests/parse-events.c +++ b/tools/perf/tests/parse-events.c @@ -109,7 +109,7 @@ static int test__checkevent_tracepoint(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVLIST("wrong number of groups", 0 == evlist__nr_groups(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_TRACEPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong sample_type", @@ -122,7 +122,7 @@ static int test__checkevent_tracepoint_multi(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", evlist->core.nr_entries > 1, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", evlist__nr_entries(evlist) > 1, evlist); TEST_ASSERT_EVLIST("wrong number of groups", 0 == evlist__nr_groups(evlist), evlist); evlist__for_each_entry(evlist, evsel) { @@ -144,7 +144,7 @@ static int test__checkevent_raw(struct evlist *evlist) struct evsel *evsel; bool raw_type_match = false; - TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { struct perf_pmu *pmu __maybe_unused = NULL; @@ -182,7 +182,7 @@ static int test__checkevent_numeric(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", 1 == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 1 == evsel->core.attr.config, evsel); return TEST_OK; @@ -193,7 +193,7 @@ static int test__checkevent_symbolic_name(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("unexpected event", @@ -207,7 +207,7 @@ static int test__checkevent_symbolic_name_config(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("unexpected event", @@ -228,7 +228,7 @@ static int test__checkevent_symbolic_alias(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type/config", evsel__match(evsel, SOFTWARE, SW_PAGE_FAULTS), evsel); return TEST_OK; @@ -238,7 +238,7 @@ static int test__checkevent_genhw(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_HW_CACHE == evsel->core.attr.type, evsel); @@ -251,7 +251,7 @@ static int test__checkevent_breakpoint(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", @@ -265,7 +265,7 @@ static int test__checkevent_breakpoint_x(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", HW_BREAKPOINT_X == evsel->core.attr.bp_type, evsel); @@ -278,7 +278,7 @@ static int test__checkevent_breakpoint_r(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", HW_BREAKPOINT_R == evsel->core.attr.bp_type, evsel); @@ -290,7 +290,7 @@ static int test__checkevent_breakpoint_w(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", HW_BREAKPOINT_W == evsel->core.attr.bp_type, evsel); @@ -302,7 +302,7 @@ static int test__checkevent_breakpoint_rw(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", @@ -316,7 +316,7 @@ static int test__checkevent_tracepoint_modifier(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong exclude_user", evsel->core.attr.exclude_user, evsel); TEST_ASSERT_EVSEL("wrong exclude_kernel", !evsel->core.attr.exclude_kernel, evsel); TEST_ASSERT_EVSEL("wrong exclude_hv", evsel->core.attr.exclude_hv, evsel); @@ -330,7 +330,7 @@ test__checkevent_tracepoint_multi_modifier(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", evlist->core.nr_entries > 1, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", evlist__nr_entries(evlist) > 1, evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel); @@ -346,7 +346,7 @@ static int test__checkevent_raw_modifier(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("wrong exclude_user", evsel->core.attr.exclude_user, evsel); @@ -361,7 +361,7 @@ static int test__checkevent_numeric_modifier(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("wrong exclude_user", evsel->core.attr.exclude_user, evsel); @@ -377,7 +377,7 @@ static int test__checkevent_symbolic_name_modifier(struct evlist *evlist) struct evsel *evsel; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { @@ -394,7 +394,7 @@ static int test__checkevent_exclude_host_modifier(struct evlist *evlist) struct evsel *evsel; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { @@ -409,7 +409,7 @@ static int test__checkevent_exclude_guest_modifier(struct evlist *evlist) struct evsel *evsel; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { @@ -423,7 +423,8 @@ static int test__checkevent_symbolic_alias_modifier(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", + 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel); TEST_ASSERT_EVSEL("wrong exclude_kernel", evsel->core.attr.exclude_kernel, evsel); TEST_ASSERT_EVSEL("wrong exclude_hv", evsel->core.attr.exclude_hv, evsel); @@ -437,7 +438,7 @@ static int test__checkevent_genhw_modifier(struct evlist *evlist) struct evsel *evsel; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { @@ -454,7 +455,7 @@ static int test__checkevent_exclude_idle_modifier(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong exclude idle", evsel->core.attr.exclude_idle, evsel); @@ -473,7 +474,7 @@ static int test__checkevent_exclude_idle_modifier_1(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong exclude idle", evsel->core.attr.exclude_idle, evsel); @@ -622,7 +623,7 @@ static int test__checkevent_breakpoint_2_events(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVSEL("wrong number of entries", 2 == evlist->core.nr_entries, evsel); + TEST_ASSERT_EVSEL("wrong number of entries", 2 == evlist__nr_entries(evlist), evsel); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong name", evsel__name_is(evsel, "breakpoint1"), evsel); @@ -641,7 +642,7 @@ static int test__checkevent_pmu(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); struct perf_pmu *core_pmu = perf_pmus__find_core_pmu(); - TEST_ASSERT_EVSEL("wrong number of entries", 1 == evlist->core.nr_entries, evsel); + TEST_ASSERT_EVSEL("wrong number of entries", 1 == evlist__nr_entries(evlist), evsel); TEST_ASSERT_EVSEL("wrong type", core_pmu->type == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", test_hw_config(evsel, 10), evsel); TEST_ASSERT_EVSEL("wrong config1", 1 == evsel->core.attr.config1, evsel); @@ -661,7 +662,7 @@ static int test__checkevent_list(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVSEL("wrong number of entries", 3 <= evlist->core.nr_entries, evsel); + TEST_ASSERT_EVSEL("wrong number of entries", 3 <= evlist__nr_entries(evlist), evsel); /* r1 */ TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_TRACEPOINT != evsel->core.attr.type, evsel); @@ -707,14 +708,15 @@ static int test__checkevent_pmu_name(struct evlist *evlist) char buf[256]; /* default_core/config=1,name=krava/u */ - TEST_ASSERT_EVLIST("wrong number of entries", 2 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", + 2 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", core_pmu->type == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 1 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong name", evsel__name_is(evsel, "krava"), evsel); /* default_core/config=2/u" */ evsel = evsel__next(evsel); - TEST_ASSERT_EVSEL("wrong number of entries", 2 == evlist->core.nr_entries, evsel); + TEST_ASSERT_EVSEL("wrong number of entries", 2 == evlist__nr_entries(evlist), evsel); TEST_ASSERT_EVSEL("wrong type", core_pmu->type == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 2 == evsel->core.attr.config, evsel); snprintf(buf, sizeof(buf), "%s/config=2/u", core_pmu->name); @@ -729,7 +731,8 @@ static int test__checkevent_pmu_partial_time_callgraph(struct evlist *evlist) struct perf_pmu *core_pmu = perf_pmus__find_core_pmu(); /* default_core/config=1,call-graph=fp,time,period=100000/ */ - TEST_ASSERT_EVLIST("wrong number of entries", 2 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", + 2 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", core_pmu->type == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 1 == evsel->core.attr.config, evsel); /* @@ -760,7 +763,7 @@ static int test__checkevent_pmu_events(struct evlist *evlist) struct evsel *evsel; struct perf_pmu *core_pmu = perf_pmus__find_core_pmu(); - TEST_ASSERT_EVLIST("wrong number of entries", 1 <= evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 <= evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("wrong type", @@ -787,8 +790,9 @@ static int test__checkevent_pmu_events_mix(struct evlist *evlist) * The wild card event will be opened at least once, but it may be * opened on each core PMU. */ - TEST_ASSERT_EVLIST("wrong number of entries", evlist->core.nr_entries >= 2, evlist); - for (int i = 0; i < evlist->core.nr_entries - 1; i++) { + TEST_ASSERT_EVLIST("wrong number of entries", + evlist__nr_entries(evlist) >= 2, evlist); + for (int i = 0; i < evlist__nr_entries(evlist) - 1; i++) { evsel = (i == 0 ? evlist__first(evlist) : evsel__next(evsel)); /* pmu-event:u */ TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel); @@ -905,7 +909,7 @@ static int test__group1(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (num_core_entries(evlist) * 2), + evlist__nr_entries(evlist) == (num_core_entries(evlist) * 2), evlist); TEST_ASSERT_EVLIST("wrong number of groups", evlist__nr_groups(evlist) == num_core_entries(evlist), @@ -950,7 +954,7 @@ static int test__group2(struct evlist *evlist) struct evsel *evsel, *leader = NULL; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (2 * num_core_entries(evlist) + 1), + evlist__nr_entries(evlist) == (2 * num_core_entries(evlist) + 1), evlist); /* * TODO: Currently the software event won't be grouped with the hardware @@ -1018,7 +1022,7 @@ static int test__group3(struct evlist *evlist __maybe_unused) struct evsel *evsel, *group1_leader = NULL, *group2_leader = NULL; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (3 * perf_pmus__num_core_pmus() + 2), + evlist__nr_entries(evlist) == (3 * perf_pmus__num_core_pmus() + 2), evlist); /* * Currently the software event won't be grouped with the hardware event @@ -1144,7 +1148,7 @@ static int test__group4(struct evlist *evlist __maybe_unused) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (num_core_entries(evlist) * 2), + evlist__nr_entries(evlist) == (num_core_entries(evlist) * 2), evlist); TEST_ASSERT_EVLIST("wrong number of groups", num_core_entries(evlist) == evlist__nr_groups(evlist), @@ -1191,7 +1195,7 @@ static int test__group5(struct evlist *evlist __maybe_unused) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (5 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (5 * num_core_entries(evlist)), evlist); TEST_ASSERT_EVLIST("wrong number of groups", evlist__nr_groups(evlist) == (2 * num_core_entries(evlist)), @@ -1284,7 +1288,7 @@ static int test__group_gh1(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (2 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)), evlist); TEST_ASSERT_EVLIST("wrong number of groups", evlist__nr_groups(evlist) == num_core_entries(evlist), @@ -1329,7 +1333,7 @@ static int test__group_gh2(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (2 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)), evlist); TEST_ASSERT_EVLIST("wrong number of groups", evlist__nr_groups(evlist) == num_core_entries(evlist), @@ -1374,7 +1378,7 @@ static int test__group_gh3(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (2 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)), evlist); TEST_ASSERT_EVLIST("wrong number of groups", evlist__nr_groups(evlist) == num_core_entries(evlist), @@ -1419,7 +1423,7 @@ static int test__group_gh4(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (2 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)), evlist); TEST_ASSERT_EVLIST("wrong number of groups", evlist__nr_groups(evlist) == num_core_entries(evlist), @@ -1464,7 +1468,7 @@ static int test__leader_sample1(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (3 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (3 * num_core_entries(evlist)), evlist); for (int i = 0; i < num_core_entries(evlist); i++) { @@ -1520,7 +1524,7 @@ static int test__leader_sample2(struct evlist *evlist __maybe_unused) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (2 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)), evlist); for (int i = 0; i < num_core_entries(evlist); i++) { @@ -1562,7 +1566,7 @@ static int test__checkevent_pinned_modifier(struct evlist *evlist) struct evsel *evsel = NULL; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); for (int i = 0; i < num_core_entries(evlist); i++) { @@ -1581,7 +1585,7 @@ static int test__pinned_group(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (3 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (3 * num_core_entries(evlist)), evlist); for (int i = 0; i < num_core_entries(evlist); i++) { @@ -1618,7 +1622,7 @@ static int test__checkevent_exclusive_modifier(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel); TEST_ASSERT_EVSEL("wrong exclude_kernel", evsel->core.attr.exclude_kernel, evsel); @@ -1634,7 +1638,7 @@ static int test__exclusive_group(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == 3 * num_core_entries(evlist), + evlist__nr_entries(evlist) == 3 * num_core_entries(evlist), evlist); for (int i = 0; i < num_core_entries(evlist); i++) { @@ -1669,7 +1673,7 @@ static int test__checkevent_breakpoint_len(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", @@ -1684,7 +1688,7 @@ static int test__checkevent_breakpoint_len_w(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", HW_BREAKPOINT_W == evsel->core.attr.bp_type, evsel); @@ -1698,7 +1702,7 @@ test__checkevent_breakpoint_len_rw_modifier(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel); TEST_ASSERT_EVSEL("wrong exclude_kernel", evsel->core.attr.exclude_kernel, evsel); TEST_ASSERT_EVSEL("wrong exclude_hv", evsel->core.attr.exclude_hv, evsel); @@ -1712,7 +1716,7 @@ static int test__checkevent_precise_max_modifier(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == 1 + num_core_entries(evlist), + evlist__nr_entries(evlist) == 1 + num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type/config", evsel__match(evsel, SOFTWARE, SW_TASK_CLOCK), evsel); return TEST_OK; @@ -1723,7 +1727,7 @@ static int test__checkevent_config_symbol(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "insn"), evsel); return TEST_OK; @@ -1733,7 +1737,7 @@ static int test__checkevent_config_raw(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "rawpmu"), evsel); return TEST_OK; } @@ -1742,7 +1746,7 @@ static int test__checkevent_config_num(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "numpmu"), evsel); return TEST_OK; } @@ -1752,7 +1756,7 @@ static int test__checkevent_config_cache(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "cachepmu"), evsel); return test__checkevent_genhw(evlist); @@ -1777,7 +1781,7 @@ static int test__intel_pt(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "intel_pt//u"), evsel); return TEST_OK; } @@ -1798,7 +1802,8 @@ static int test__ratio_to_prev(struct evlist *evlist) { struct evsel *evsel, *leader; - TEST_ASSERT_VAL("wrong number of entries", 2 * perf_pmus__num_core_pmus() == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", + 2 * perf_pmus__num_core_pmus() == evlist__nr_entries(evlist)); evlist__for_each_entry(evlist, evsel) { if (evsel != evsel__leader(evsel) || @@ -1842,7 +1847,7 @@ static int test__checkevent_complex_name(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong complex name parsing", evsel__name_is(evsel, @@ -1855,7 +1860,7 @@ static int test__checkevent_raw_pmu(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_SOFTWARE == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0x1a == evsel->core.attr.config, evsel); return TEST_OK; @@ -1866,7 +1871,7 @@ static int test__sym_event_slash(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("unexpected event", evsel__match(evsel, HARDWARE, HW_CPU_CYCLES), evsel); TEST_ASSERT_EVSEL("wrong exclude_kernel", evsel->core.attr.exclude_kernel, evsel); @@ -1878,7 +1883,7 @@ static int test__sym_event_dc(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("unexpected event", evsel__match(evsel, HARDWARE, HW_CPU_CYCLES), evsel); TEST_ASSERT_EVSEL("wrong exclude_user", evsel->core.attr.exclude_user, evsel); @@ -1890,7 +1895,7 @@ static int test__term_equal_term(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("unexpected event", evsel__match(evsel, HARDWARE, HW_CPU_CYCLES), evsel); TEST_ASSERT_EVSEL("wrong name setting", strcmp(evsel->name, "name") == 0, evsel); @@ -1902,7 +1907,7 @@ static int test__term_equal_legacy(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("unexpected event", evsel__match(evsel, HARDWARE, HW_CPU_CYCLES), evsel); TEST_ASSERT_EVSEL("wrong name setting", strcmp(evsel->name, "l1d") == 0, evsel); @@ -1958,7 +1963,7 @@ static int count_tracepoints(void) static int test__all_tracepoints(struct evlist *evlist) { TEST_ASSERT_VAL("wrong events count", - count_tracepoints() == evlist->core.nr_entries); + count_tracepoints() == evlist__nr_entries(evlist)); return test__checkevent_tracepoint_multi(evlist); } @@ -2556,8 +2561,10 @@ static int test_event(const struct evlist_test *e) return TEST_FAIL; } parse_events_error__init(&err); - ret = __parse_events(evlist, e->name, /*pmu_filter=*/NULL, &err, /*fake_pmu=*/false, - /*warn_if_reordered=*/true, /*fake_tp=*/true); + ret = __parse_events(evlist, e->name, /*pmu_filter=*/NULL, + /*cputype_filter=*/false, &err, /*fake_pmu=*/false, + /*warn_if_reordered=*/true, + /*fake_tp=*/true); if (ret) { pr_debug("failed to parse event '%s', err %d\n", e->name, ret); parse_events_error__print(&err, e->name); @@ -2568,7 +2575,7 @@ static int test_event(const struct evlist_test *e) ret = e->check(evlist); } parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); return ret; } @@ -2584,8 +2591,9 @@ static int test_event_fake_pmu(const char *str) return -ENOMEM; parse_events_error__init(&err); - ret = __parse_events(evlist, str, /*pmu_filter=*/NULL, &err, - /*fake_pmu=*/true, /*warn_if_reordered=*/true, + ret = __parse_events(evlist, str, /*pmu_filter=*/NULL, + /*cputype_filter=*/false, &err, /*fake_pmu=*/true, + /*warn_if_reordered=*/true, /*fake_tp=*/true); if (ret) { pr_debug("failed to parse event '%s', err %d\n", @@ -2594,7 +2602,7 @@ static int test_event_fake_pmu(const char *str) } parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); return ret; } diff --git a/tools/perf/tests/parse-metric.c b/tools/perf/tests/parse-metric.c index 7c7f489a5eb0a6..872d9a7aa72f65 100644 --- a/tools/perf/tests/parse-metric.c +++ b/tools/perf/tests/parse-metric.c @@ -53,7 +53,7 @@ static double compute_single(struct evlist *evlist, const char *name) struct evsel *evsel; evlist__for_each_entry(evlist, evsel) { - me = metricgroup__lookup(&evlist->metric_events, evsel, false); + me = metricgroup__lookup(evlist__metric_events(evlist), evsel, false); if (me != NULL) { list_for_each_entry (mexp, &me->head, nd) { if (strcmp(mexp->metric_name, name)) @@ -84,15 +84,16 @@ static int __compute_metric(const char *name, struct value *vals, cpus = perf_cpu_map__new("0"); if (!cpus) { - evlist__delete(evlist); + evlist__put(evlist); return -ENOMEM; } - perf_evlist__set_maps(&evlist->core, cpus, NULL); + perf_evlist__set_maps(evlist__core(evlist), cpus, NULL); /* Parse the metric into metric_events list. */ pme_test = find_core_metrics_table("testarch", "testcpu"); - err = metricgroup__parse_groups_test(evlist, pme_test, name); + err = metricgroup__parse_groups_test(evlist, pme_test, name, + /*cputype_filter=*/false); if (err) goto out; @@ -113,7 +114,7 @@ out: /* ... cleanup. */ evlist__free_stats(evlist); perf_cpu_map__put(cpus); - evlist__delete(evlist); + evlist__put(evlist); return err; } diff --git a/tools/perf/tests/parse-no-sample-id-all.c b/tools/perf/tests/parse-no-sample-id-all.c index 8ac862c94879f3..78bb8db1924043 100644 --- a/tools/perf/tests/parse-no-sample-id-all.c +++ b/tools/perf/tests/parse-no-sample-id-all.c @@ -49,7 +49,7 @@ static int process_events(union perf_event **events, size_t count) for (i = 0; i < count && !err; i++) err = process_event(&evlist, events[i]); - evlist__delete(evlist); + evlist__put(evlist); return err; } diff --git a/tools/perf/tests/perf-record.c b/tools/perf/tests/perf-record.c index ad44cc68820b30..0cac6ae1a1fc05 100644 --- a/tools/perf/tests/perf-record.c +++ b/tools/perf/tests/perf-record.c @@ -105,7 +105,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest err = evlist__create_maps(evlist, &opts.target); if (err < 0) { pr_debug("Not enough memory to create thread/cpu maps\n"); - goto out_delete_evlist; + goto out_put_evlist; } /* @@ -117,7 +117,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest err = evlist__prepare_workload(evlist, &opts.target, argv, false, NULL); if (err < 0) { pr_debug("Couldn't run the workload!\n"); - goto out_delete_evlist; + goto out_put_evlist; } /* @@ -129,12 +129,12 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest evsel__set_sample_bit(evsel, TIME); evlist__config(evlist, &opts, NULL); - err = sched__get_first_possible_cpu(evlist->workload.pid, cpu_mask); + err = sched__get_first_possible_cpu(evlist__workload_pid(evlist), cpu_mask); if (err < 0) { pr_debug("sched__get_first_possible_cpu: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); evlist__cancel_workload(evlist); - goto out_delete_evlist; + goto out_put_evlist; } cpu = err; @@ -142,11 +142,11 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest /* * So that we can check perf_sample.cpu on all the samples. */ - if (sched_setaffinity(evlist->workload.pid, cpu_mask_size, cpu_mask) < 0) { + if (sched_setaffinity(evlist__workload_pid(evlist), cpu_mask_size, cpu_mask) < 0) { pr_debug("sched_setaffinity: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); evlist__cancel_workload(evlist); - goto out_delete_evlist; + goto out_put_evlist; } /* @@ -158,7 +158,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest pr_debug("perf_evlist__open: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); evlist__cancel_workload(evlist); - goto out_delete_evlist; + goto out_put_evlist; } /* @@ -166,12 +166,12 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest * fds in the same CPU to be injected in the same mmap ring buffer * (using ioctl(PERF_EVENT_IOC_SET_OUTPUT)). */ - err = evlist__mmap(evlist, opts.mmap_pages); + err = evlist__do_mmap(evlist, opts.mmap_pages); if (err < 0) { pr_debug("evlist__mmap: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); evlist__cancel_workload(evlist); - goto out_delete_evlist; + goto out_put_evlist; } /* @@ -188,11 +188,11 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest while (1) { int before = total_events; - for (i = 0; i < evlist->core.nr_mmaps; i++) { + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { union perf_event *event; struct mmap *md; - md = &evlist->mmap[i]; + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; @@ -209,7 +209,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest if (verbose > 0) perf_event__fprintf(event, NULL, stderr); pr_debug("Couldn't parse sample\n"); - goto out_delete_evlist; + goto out_put_evlist; } if (verbose > 0) { @@ -231,15 +231,15 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest ++errs; } - if ((pid_t)sample.pid != evlist->workload.pid) { + if ((pid_t)sample.pid != evlist__workload_pid(evlist)) { pr_debug("%s with unexpected pid, expected %d, got %d\n", - name, evlist->workload.pid, sample.pid); + name, evlist__workload_pid(evlist), sample.pid); ++errs; } - if ((pid_t)sample.tid != evlist->workload.pid) { + if ((pid_t)sample.tid != evlist__workload_pid(evlist)) { pr_debug("%s with unexpected tid, expected %d, got %d\n", - name, evlist->workload.pid, sample.tid); + name, evlist__workload_pid(evlist), sample.tid); ++errs; } @@ -248,7 +248,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest type == PERF_RECORD_MMAP2 || type == PERF_RECORD_FORK || type == PERF_RECORD_EXIT) && - (pid_t)event->comm.pid != evlist->workload.pid) { + (pid_t)event->comm.pid != evlist__workload_pid(evlist)) { pr_debug("%s with unexpected pid/tid\n", name); ++errs; } @@ -350,11 +350,11 @@ found_exit: pr_debug("PERF_RECORD_MMAP for %s missing!\n", "[vdso]"); ++errs; } -out_delete_evlist: +out_put_evlist: CPU_FREE(cpu_mask); - evlist__delete(evlist); out: perf_sample__exit(&sample); + evlist__put(evlist); if (err == -EACCES) return TEST_SKIP; if (err < 0 || errs != 0) diff --git a/tools/perf/tests/perf-time-to-tsc.c b/tools/perf/tests/perf-time-to-tsc.c index cca41bd37ae3c2..f8f71fdd32b1bf 100644 --- a/tools/perf/tests/perf-time-to-tsc.c +++ b/tools/perf/tests/perf-time-to-tsc.c @@ -99,7 +99,7 @@ static int test__perf_time_to_tsc(struct test_suite *test __maybe_unused, int su evlist = evlist__new(); CHECK_NOT_NULL__(evlist); - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); CHECK__(parse_event(evlist, "cpu-cycles:u")); @@ -121,9 +121,9 @@ static int test__perf_time_to_tsc(struct test_suite *test __maybe_unused, int su goto out_err; } - CHECK__(evlist__mmap(evlist, UINT_MAX)); + CHECK__(evlist__do_mmap(evlist, UINT_MAX)); - pc = evlist->mmap[0].core.base; + pc = evlist__mmap(evlist)[0].core.base; ret = perf_read_tsc_conversion(pc, &tc); if (ret) { if (ret == -EOPNOTSUPP) { @@ -145,8 +145,8 @@ static int test__perf_time_to_tsc(struct test_suite *test __maybe_unused, int su evlist__disable(evlist); - for (i = 0; i < evlist->core.nr_mmaps; i++) { - md = &evlist->mmap[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; @@ -201,7 +201,7 @@ next_event: err = TEST_OK; out_err: - evlist__delete(evlist); + evlist__put(evlist); perf_cpu_map__put(cpus); perf_thread_map__put(threads); return err; diff --git a/tools/perf/tests/pfm.c b/tools/perf/tests/pfm.c index fca4a86452df63..f7bf55be5e6ee8 100644 --- a/tools/perf/tests/pfm.c +++ b/tools/perf/tests/pfm.c @@ -69,18 +69,18 @@ static int test__pfm_events(struct test_suite *test __maybe_unused, if (evlist == NULL) return -ENOMEM; - opt.value = evlist; + opt.value = &evlist; parse_libpfm_events_option(&opt, table[i].events, 0); TEST_ASSERT_EQUAL(table[i].events, - count_pfm_events(&evlist->core), + count_pfm_events(evlist__core(evlist)), table[i].nr_events); TEST_ASSERT_EQUAL(table[i].events, evlist__nr_groups(evlist), 0); - evlist__delete(evlist); + evlist__put(evlist); } return 0; } @@ -154,18 +154,18 @@ static int test__pfm_group(struct test_suite *test __maybe_unused, if (evlist == NULL) return -ENOMEM; - opt.value = evlist; + opt.value = &evlist; parse_libpfm_events_option(&opt, table[i].events, 0); TEST_ASSERT_EQUAL(table[i].events, - count_pfm_events(&evlist->core), + count_pfm_events(evlist__core(evlist)), table[i].nr_events); TEST_ASSERT_EQUAL(table[i].events, evlist__nr_groups(evlist), table[i].nr_groups); - evlist__delete(evlist); + evlist__put(evlist); } return 0; } diff --git a/tools/perf/tests/pmu-events.c b/tools/perf/tests/pmu-events.c index fd5630f0a13c07..f507ce39439ad8 100644 --- a/tools/perf/tests/pmu-events.c +++ b/tools/perf/tests/pmu-events.c @@ -794,11 +794,13 @@ static int check_parse_id(const char *id, struct parse_events_error *error) for (cur = strchr(dup, '@') ; cur; cur = strchr(++cur, '@')) *cur = '/'; - ret = __parse_events(evlist, dup, /*pmu_filter=*/NULL, error, /*fake_pmu=*/true, - /*warn_if_reordered=*/true, /*fake_tp=*/false); + ret = __parse_events(evlist, dup, /*pmu_filter=*/NULL, + /*cputype_filter=*/false, error, /*fake_pmu=*/true, + /*warn_if_reordered=*/true, + /*fake_tp=*/false); free(dup); - evlist__delete(evlist); + evlist__put(evlist); return ret; } @@ -865,13 +867,15 @@ static int test__parsing_callback(const struct pmu_metric *pm, cpus = perf_cpu_map__new("0"); if (!cpus) { - evlist__delete(evlist); + evlist__put(evlist); return -ENOMEM; } - perf_evlist__set_maps(&evlist->core, cpus, NULL); + perf_evlist__set_maps(evlist__core(evlist), cpus, NULL); - err = metricgroup__parse_groups_test(evlist, table, pm->metric_name); + err = metricgroup__parse_groups_test(evlist, table, + pm->metric_name, + /*cputype_filter=*/false); if (err) { if (is_expected_broken_metric(pm)) { (*failures)--; @@ -895,7 +899,8 @@ static int test__parsing_callback(const struct pmu_metric *pm, k++; } evlist__for_each_entry(evlist, evsel) { - struct metric_event *me = metricgroup__lookup(&evlist->metric_events, evsel, false); + struct metric_event *me = metricgroup__lookup(evlist__metric_events(evlist), + evsel, false); if (me != NULL) { struct metric_expr *mexp; @@ -919,7 +924,7 @@ out_err: /* ... cleanup. */ evlist__free_stats(evlist); perf_cpu_map__put(cpus); - evlist__delete(evlist); + evlist__put(evlist); return err; } diff --git a/tools/perf/tests/pmu.c b/tools/perf/tests/pmu.c index d7be9d1c6f52b8..13e8d7fa80af2a 100644 --- a/tools/perf/tests/pmu.c +++ b/tools/perf/tests/pmu.c @@ -294,7 +294,7 @@ static int test__pmu_config_helpers(struct test_suite *test __maybe_unused, ret = TEST_OK; err_out: parse_events_terms__exit(&terms); - evlist__delete(evlist); + evlist__put(evlist); test_pmu_put(dir, pmu); return ret; } @@ -346,7 +346,7 @@ static int test__pmu_events(struct test_suite *test __maybe_unused, int subtest ret = TEST_OK; err_out: parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); test_pmu_put(dir, pmu); return ret; } diff --git a/tools/perf/tests/sample-parsing.c b/tools/perf/tests/sample-parsing.c index 55f0b73ca20e05..20cab91ceaeb2f 100644 --- a/tools/perf/tests/sample-parsing.c +++ b/tools/perf/tests/sample-parsing.c @@ -205,15 +205,11 @@ static bool samples_same(struct perf_sample *s1, static int do_test(u64 sample_type, u64 sample_regs, u64 read_format) { - struct evsel evsel = { - .needs_swap = false, - .core = { - . attr = { - .sample_type = sample_type, - .read_format = read_format, - }, - }, + struct perf_event_attr attr = { + .sample_type = sample_type, + .read_format = read_format, }; + struct evsel *evsel; union perf_event *event; union { struct ip_callchain callchain; @@ -287,16 +283,21 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format) size_t i, sz, bufsz; int err, ret = -1; + evsel = evsel__new(&attr); + if (!evsel) { + pr_debug("evsel__new failed\n"); + return -1; + } perf_sample__init(&sample_out, /*all=*/false); perf_sample__init(&sample_out_endian, /*all=*/false); if (sample_type & PERF_SAMPLE_REGS_USER) - evsel.core.attr.sample_regs_user = sample_regs; + evsel->core.attr.sample_regs_user = sample_regs; if (sample_type & PERF_SAMPLE_REGS_INTR) - evsel.core.attr.sample_regs_intr = sample_regs; + evsel->core.attr.sample_regs_intr = sample_regs; if (sample_type & PERF_SAMPLE_BRANCH_STACK) - evsel.core.attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; + evsel->core.attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; for (i = 0; i < sizeof(regs); i++) *(i + (u8 *)regs) = i & 0xfe; @@ -311,12 +312,12 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format) } sz = perf_event__sample_event_size(&sample, sample_type, read_format, - evsel.core.attr.branch_sample_type); + evsel->core.attr.branch_sample_type); bufsz = sz + 4096; /* Add a bit for overrun checking */ event = malloc(bufsz); if (!event) { pr_debug("malloc failed\n"); - return -1; + goto out_free; } memset(event, 0xff, bufsz); @@ -325,7 +326,7 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format) event->header.size = sz; err = perf_event__synthesize_sample(event, sample_type, read_format, - evsel.core.attr.branch_sample_type, &sample); + evsel->core.attr.branch_sample_type, &sample); if (err) { pr_debug("%s failed for sample_type %#"PRIx64", error %d\n", "perf_event__synthesize_sample", sample_type, err); @@ -343,32 +344,33 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format) goto out_free; } - evsel.sample_size = __evsel__sample_size(sample_type); + evsel->sample_size = __evsel__sample_size(sample_type); - err = evsel__parse_sample(&evsel, event, &sample_out); + err = evsel__parse_sample(evsel, event, &sample_out); if (err) { pr_debug("%s failed for sample_type %#"PRIx64", error %d\n", "evsel__parse_sample", sample_type, err); goto out_free; } - if (!samples_same(&sample, &sample_out, sample_type, read_format, evsel.needs_swap)) { + if (!samples_same(&sample, &sample_out, sample_type, read_format, evsel->needs_swap)) { pr_debug("parsing failed for sample_type %#"PRIx64"\n", sample_type); goto out_free; } if (sample_type == PERF_SAMPLE_BRANCH_STACK) { - evsel.needs_swap = true; - evsel.sample_size = __evsel__sample_size(sample_type); - err = evsel__parse_sample(&evsel, event, &sample_out_endian); + evsel->needs_swap = true; + evsel->sample_size = __evsel__sample_size(sample_type); + err = evsel__parse_sample(evsel, event, &sample_out_endian); if (err) { pr_debug("%s failed for sample_type %#"PRIx64", error %d\n", "evsel__parse_sample", sample_type, err); goto out_free; } - if (!samples_same(&sample, &sample_out_endian, sample_type, read_format, evsel.needs_swap)) { + if (!samples_same(&sample, &sample_out_endian, sample_type, + read_format, evsel->needs_swap)) { pr_debug("parsing failed for sample_type %#"PRIx64"\n", sample_type); goto out_free; @@ -380,6 +382,7 @@ out_free: free(event); perf_sample__exit(&sample_out_endian); perf_sample__exit(&sample_out); + evsel__put(evsel); if (ret && read_format) pr_debug("read_format %#"PRIx64"\n", read_format); return ret; diff --git a/tools/perf/tests/shell/coresight/test_arm_coresight_disasm.sh b/tools/perf/tests/shell/coresight/test_arm_coresight_disasm.sh index ccb90dda247585..f3ebad5963783e 100755 --- a/tools/perf/tests/shell/coresight/test_arm_coresight_disasm.sh +++ b/tools/perf/tests/shell/coresight/test_arm_coresight_disasm.sh @@ -44,7 +44,7 @@ branch_search='[[:space:]](bl|b(\.(eq|ne|cs|cc|mi|pl|vs|vc|hi|ls|ge|lt|gt|le|al) if [ "$(id -u)" == 0 ] && [ -e /proc/kcore ]; then echo "Testing kernel disassembly" perf record -o ${perfdata} -e cs_etm//k --kcore -Se -m,64K -- touch $file > /dev/null 2>&1 - perf script -i ${perfdata} -s python:${script_path} -- \ + perf script -i ${perfdata} --itrace=b -s python:${script_path} -- \ -d --stop-sample=2 -k ${perfdata}/kcore_dir/kcore 2> /dev/null > ${file} grep -q -E ${branch_search} ${file} echo "Found kernel branches" @@ -56,7 +56,7 @@ fi ## Test user ## echo "Testing userspace disassembly" perf record -o ${perfdata} -e cs_etm//u -Se -m,64K -- touch $file > /dev/null 2>&1 -perf script -i ${perfdata} -s python:${script_path} -- \ +perf script -i ${perfdata} --itrace=b -s python:${script_path} -- \ -d --stop-sample=2 2> /dev/null > ${file} grep -q -E ${branch_search} ${file} echo "Found userspace branches" diff --git a/tools/perf/tests/shell/inject_aslr.sh b/tools/perf/tests/shell/inject_aslr.sh index c00461828ea796..d83c2591db8f69 100755 --- a/tools/perf/tests/shell/inject_aslr.sh +++ b/tools/perf/tests/shell/inject_aslr.sh @@ -135,8 +135,14 @@ test_callchain_aslr() { echo "Callchain ASLR test [Failed - no noploop samples in original file]" err=1 elif [ -z "$new_addr" ]; then - echo "Callchain ASLR test [Failed - could not find remapped address]" - err=1 + if perf evlist -v -i "${data}" | grep -q 'sample_type:.*STACK_USER'; then + echo "Dropping stack user sample as possible ASLR leak" + echo "Call-graph dwarf not supported with 'perf inject --aslr'" + echo "Callchain ASLR test [Skip]" + else + echo "Callchain ASLR test [Failed - could not find remapped address]" + err=1 + fi elif [ "$orig_addr" = "$new_addr" ]; then echo "Callchain ASLR test [Failed - addresses are not remapped]" err=1 diff --git a/tools/perf/tests/shell/jitdump-python.sh b/tools/perf/tests/shell/jitdump-python.sh index ae86203b14a22b..05aaa3bd900b52 100755 --- a/tools/perf/tests/shell/jitdump-python.sh +++ b/tools/perf/tests/shell/jitdump-python.sh @@ -16,11 +16,15 @@ if [ "${HAS_PERF_JIT}" != "True" ]; then exit 2 fi -PERF_DATA=$(mktemp /tmp/__perf_test.perf.data.XXXXXX) +PERF_DATA_DIR=$(mktemp -d /tmp/__perf_test.perf.data.dir.XXXXXX) +PERF_DATA="${PERF_DATA_DIR}/perf.data" cleanup() { echo "Cleaning up files..." - rm -f ${PERF_DATA} ${PERF_DATA}.jit /tmp/jit-${PID}.dump /tmp/jitted-${PID}-*.so 2> /dev/null + rm -rf ${PERF_DATA_DIR} 2> /dev/null + for p in ${ALL_PIDS}; do + rm -f /tmp/jit-${p}.dump /tmp/jitted-${p}-*.so 2> /dev/null + done trap - EXIT TERM INT } @@ -33,9 +37,16 @@ trap_cleanup() { trap trap_cleanup EXIT TERM INT -echo "Run python with -Xperf_jit" -cat <') - -echo "Found ${NUM} matching lines" - -echo "Remove JIT-ed DSOs from the build-ID cache" -for F in /tmp/jitted-${PID}-*.so; do - perf buildid-cache -r "${F}" + if [ -f "${PERF_DATA}.pid" ]; then + REAL_PID=$(cat "${PERF_DATA}.pid") + ALL_PIDS="${ALL_PIDS} ${REAL_PID}" + fi + + # extract PID of the target process from the data + PID=$(perf report -i "${PERF_DATA}" --stdio -F pid -q -g none | \ + cut -d: -f1 -s | sort -u | head -n 1 | tr -d ' ') + if [ -z "${PID}" ]; then + echo "Failed to get PID, retrying..." + continue + fi + ALL_PIDS="${ALL_PIDS} ${PID}" + + echo "Generate JIT-ed DSOs using perf inject" + DEBUGINFOD_URLS='' perf inject -i "${PERF_DATA}" -j -o "${PERF_DATA}.jit" + + echo "Add JIT-ed DSOs to the build-ID cache" + for F in /tmp/jitted-${PID}-*.so; do + perf buildid-cache -a "${F}" + done + + echo "Check the symbol containing the function/module name" + NUM=$(perf report -i "${PERF_DATA}.jit" -s sym --stdio | grep -cE 'py::(foo|bar|baz):') + + echo "Remove JIT-ed DSOs from the build-ID cache" + for F in /tmp/jitted-${PID}-*.so; do + perf buildid-cache -r "${F}" + done + rm -f /tmp/jitted-${PID}-*.so /tmp/jit-${PID}.dump 2>/dev/null + + if [ "${NUM}" -gt 0 ]; then + echo "Success: found ${NUM} matching lines" + break + fi + echo "No matching lines found, retrying with more iterations..." done cleanup diff --git a/tools/perf/tests/shell/kvm.sh b/tools/perf/tests/shell/kvm.sh index f88e859025c42c..a5396f8e6fe5ca 100755 --- a/tools/perf/tests/shell/kvm.sh +++ b/tools/perf/tests/shell/kvm.sh @@ -39,17 +39,28 @@ skip() { test_kvm_stat() { echo "Testing perf kvm stat" - echo "Recording kvm events for pid ${qemu_pid}..." - if ! perf kvm stat record -p "${qemu_pid}" -o "${perfdata}" sleep 1; then - echo "Failed to record kvm events" - err=1 - return - fi + local duration + local success=false + for duration in 1 2 4 8; do + echo "Recording kvm events for pid ${qemu_pid} (duration ${duration}s)..." + rm -f "${perfdata}" "${perfdata}".old + if ! perf kvm stat record -p "${qemu_pid}" -o "${perfdata}" \ + sleep ${duration} >/dev/null 2>&1; then + echo "perf kvm stat record failed, retrying..." + continue + fi + + if [ -e "${perfdata}" ] && \ + perf kvm -i "${perfdata}" stat report 2>&1 | grep -q "VM-EXIT"; then + success=true + break + fi + echo "No VM-EXIT events found, retrying..." + done - echo "Reporting kvm events..." - if ! perf kvm -i "${perfdata}" stat report 2>&1 | grep -q "VM-EXIT"; then + if [ "$success" = false ]; then echo "Failed to find VM-EXIT in report" - perf kvm -i "${perfdata}" stat report 2>&1 + perf kvm -i "${perfdata}" stat report 2>&1 || true err=1 return fi @@ -60,22 +71,26 @@ test_kvm_stat() { test_kvm_record_report() { echo "Testing perf kvm record/report" - echo "Recording kvm profile for pid ${qemu_pid}..." - # Use --host to avoid needing guest symbols/mounts for this simple test - # We just want to verify the command runs and produces data - # We run in background and kill it because 'perf kvm record' appends options - # after the command, which breaks 'sleep' (e.g. it gets '-e cycles'). - perf kvm --host record -p "${qemu_pid}" -o "${perfdata}" & - rec_pid=$! - sleep 1 - kill -INT "${rec_pid}" - wait "${rec_pid}" || true + local duration + local success=false + for duration in 1 2 4 8; do + echo "Recording kvm profile for pid ${qemu_pid} (duration ${duration}s)..." + rm -f "${perfdata}" "${perfdata}".old + + perf kvm --host record -p "${qemu_pid}" -o "${perfdata}" \ + -e cpu-clock sleep ${duration} + + if [ -e "${perfdata}" ] && \ + perf kvm -i "${perfdata}" report --stdio 2>&1 | grep -q "Event count"; then + success=true + break + fi + echo "No samples or report failed, retrying..." + done - echo "Reporting kvm profile..." - # Check for some standard output from report - if ! perf kvm -i "${perfdata}" report --stdio 2>&1 | grep -q "Event count"; then + if [ "$success" = false ]; then echo "Failed to report kvm profile" - perf kvm -i "${perfdata}" report --stdio 2>&1 + perf kvm -i "${perfdata}" report --stdio 2>&1 || true err=1 return fi diff --git a/tools/perf/tests/shell/lib/perf_metric_validation.py b/tools/perf/tests/shell/lib/perf_metric_validation.py index dea8ef1977bf6c..3d52f94f22b914 100644 --- a/tools/perf/tests/shell/lib/perf_metric_validation.py +++ b/tools/perf/tests/shell/lib/perf_metric_validation.py @@ -383,10 +383,13 @@ class Validator: wl = workload.split() command.extend(wl) print(" ".join(command)) - cmd = subprocess.run(command, stderr=subprocess.PIPE, encoding='utf-8') - data = [x+'}' for x in cmd.stderr.split('}\n') if x] - if data[0][0] != '{': - data[0] = data[0][data[0].find('{'):] + cmd = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf-8') + lines = cmd.stderr.splitlines() + cmd.stdout.splitlines() + data = [] + for line in lines: + line = line.strip() + if line.startswith('{') and line.endswith('}'): + data.append(line) return data def collect_perf(self, workload: str): diff --git a/tools/perf/tests/shell/lib/perf_record.sh b/tools/perf/tests/shell/lib/perf_record.sh new file mode 100644 index 00000000000000..2b9e11b66dc7a8 --- /dev/null +++ b/tools/perf/tests/shell/lib/perf_record.sh @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: GPL-2.0 + +PERF_RECORD_LOGS=() + +perf_record_with_retry() { + local perfdata="$1" + local check_cmd="$2" + local testprog_base="$3" + shift 3 + + local logfile + logfile=$(mktemp /tmp/__perf_record_retry.XXXXXX) + PERF_RECORD_LOGS+=("$logfile") + + # Save the e flag state and disable it + local save_e + if [[ $- == *e* ]]; then + save_e="set -e" + else + save_e="set +e" + fi + set +e + + local duration + local first_run=true + local ret=1 + local cmd_prefix="perf record" + if [ -n "${PERF_RECORD_CMD}" ]; then + cmd_prefix="${PERF_RECORD_CMD}" + fi + + for duration in 0.01 0.1 0.3 1.0 2.0; do + rm -f "${perfdata}".old + ${cmd_prefix} "$@" -o "${perfdata}" ${testprog_base} ${duration} > "$logfile" 2>&1 + local record_exit=$? + + if [ "$first_run" = true ] && [ $record_exit -ne 0 ]; then + ret=2 + break + fi + first_run=false + + if [ -e "${perfdata}" ] && eval "${check_cmd}"; then + ret=0 + break + fi + done + + eval "$save_e" + return $ret +} + +perf_record_cleanup() { + for logfile in "${PERF_RECORD_LOGS[@]}"; do + rm -f "$logfile" + done + PERF_RECORD_LOGS=() +} diff --git a/tools/perf/tests/shell/pipe_test.sh b/tools/perf/tests/shell/pipe_test.sh index e459aa99a9515b..ce68d850c98380 100755 --- a/tools/perf/tests/shell/pipe_test.sh +++ b/tools/perf/tests/shell/pipe_test.sh @@ -12,8 +12,8 @@ skip_test_missing_symbol ${sym} data=$(mktemp /tmp/perf.data.XXXXXX) data2=$(mktemp /tmp/perf.data2.XXXXXX) -prog="perf test -w noploop" -[ "$(uname -m)" = "s390x" ] && prog="$prog 3" +prog="perf test -w noploop 0.1" +[ "$(uname -m)" = "s390x" ] && prog="perf test -w noploop 3" err=0 set -e diff --git a/tools/perf/tests/shell/record+zstd_comp_decomp_multi_record.sh b/tools/perf/tests/shell/record+zstd_comp_decomp_multi_record.sh new file mode 100755 index 00000000000000..c05ace8214ca3e --- /dev/null +++ b/tools/perf/tests/shell/record+zstd_comp_decomp_multi_record.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Zstd perf.data compression/decompression of multi-record data +# SPDX-License-Identifier: GPL-2.0 + +perfdata=$(mktemp /tmp/__perf_test.perf.data.XXXXX) +recout=$(mktemp /tmp/__perf_test.zstd.rec.XXXXX) +injout=$(mktemp /tmp/__perf_test.zstd.inj.XXXXX) +perf_tool=perf + +cleanup() { + rm -f "${perfdata}" "${perfdata}".old "${perfdata}".decomp "${recout}" "${injout}" +} +trap cleanup EXIT TERM INT + +skip_if_no_z_record() { + $perf_tool record -h 2>&1 | grep -q -- '-z, --compression-level' +} + +collect_z_record() { + echo "Collecting compressed record file:" + [ "$(uname -m)" != s390x ] && gflag='-g' + $perf_tool record -o "${perfdata}" $gflag -z -F max -m 32M --per-thread -- \ + $perf_tool test -w thloop 5 1 \ + >/dev/null 2>"${recout}" +} + +check_record() { + echo "Checking record did not fail to write data:" + if grep -q "failed to write perf data" "${recout}"; then + cat "${recout}" + return 1 + fi +} + +check_decompress() { + echo "Checking compressed file decompresses cleanly:" + if ! $perf_tool inject -i "${perfdata}" -o "${perfdata}".decomp 2>"${injout}"; then + cat "${injout}" + return 1 + fi + if grep -Eqi "decompress|corrupt|failed to process type" "${injout}"; then + cat "${injout}" + return 1 + fi +} + +skip_if_no_z_record || exit 2 +collect_z_record +check_record || exit 1 + +# Need >1 record, else the multi-record path wasn't exercised. +# Skip rather than pass/fail spuriously. +nr=$($perf_tool report -i "${perfdata}" --stats 2>/dev/null | + awk '/COMPRESSED2 events:/ { print $3 }') +if [ -z "${nr}" ] || [ "${nr}" -lt 2 ]; then + echo "less than two compressed records (${nr:-0}), skipping" + exit 2 +fi +echo "Produced ${nr} compressed records" + +check_decompress +err=$? +exit $err diff --git a/tools/perf/tests/shell/record.sh b/tools/perf/tests/shell/record.sh index 7cb81cf3444a73..dd90fef2088b1f 100755 --- a/tools/perf/tests/shell/record.sh +++ b/tools/perf/tests/shell/record.sh @@ -1,10 +1,13 @@ #!/bin/bash -# perf record tests (exclusive) # SPDX-License-Identifier: GPL-2.0 +# perf record tests set -e shelldir=$(dirname "$0") +. "${shelldir}"/lib/perf_record.sh + + # shellcheck source=lib/waiting.sh . "${shelldir}"/lib/waiting.sh @@ -39,6 +42,7 @@ cleanup() { rm -f "${perfdata}" rm -f "${perfdata}".old rm -f "${script_output}" + perf_record_cleanup trap - EXIT TERM INT } @@ -50,22 +54,20 @@ trap_cleanup() { } trap trap_cleanup EXIT TERM INT +check_per_thread() { + perf report -i "${perfdata}" -q | grep -q "${testsym}" +} + test_per_thread() { echo "Basic --per-thread mode test" - if ! perf record -o /dev/null --quiet ${testprog} 2> /dev/null - then + local ret=0 + perf_record_with_retry "${perfdata}" "check_per_thread" "perf test -w thloop" \ + --per-thread || ret=$? + if [ $ret -eq 2 ]; then echo "Per-thread record [Skipped event not supported]" return - fi - if ! perf record --per-thread -o "${perfdata}" ${testprog} 2> /dev/null - then - echo "Per-thread record [Failed record]" - err=1 - return - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then - echo "Per-thread record [Failed missing output]" + elif [ $ret -eq 1 ]; then + echo "Per-thread record [Failed record or missing output]" err=1 return fi @@ -96,6 +98,10 @@ test_per_thread() { echo "Basic --per-thread mode test [Success]" } +check_register_capture() { + perf script -F ip,sym,iregs -i "${perfdata}" 2>/dev/null | grep -q "DI:" +} + test_register_capture() { echo "Register capture test" if ! perf list pmu | grep -q 'br_inst_retired.near_call' @@ -108,11 +114,12 @@ test_register_capture() { echo "Register capture test [Skipped missing registers]" return fi - if ! perf record -o - --intr-regs=di,r8,dx,cx -e br_inst_retired.near_call \ - -c 1000 --per-thread ${testprog} 2> /dev/null \ - | perf script -F ip,sym,iregs -i - 2> /dev/null \ - | grep -q "DI:" - then + + local ret=0 + perf_record_with_retry "${perfdata}" "check_register_capture" "perf test -w thloop" \ + --intr-regs=di,r8,dx,cx -e br_inst_retired.near_call -c 1000 --per-thread || ret=$? + + if [ $ret -ne 0 ]; then echo "Register capture test [Failed missing output]" err=1 return @@ -120,65 +127,66 @@ test_register_capture() { echo "Register capture test [Success]" } +check_system_wide() { + perf report -i "${perfdata}" -q | grep -q "${testsym}" +} + test_system_wide() { echo "Basic --system-wide mode test" - if ! perf record -aB --synth=no -o "${perfdata}" ${testprog} 2> /dev/null - then + local ret=0 + perf_record_with_retry "${perfdata}" "check_system_wide" "perf test -w thloop" \ + -aB --synth=no || ret=$? + if [ $ret -eq 2 ]; then echo "System-wide record [Skipped not supported]" return - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then + elif [ $ret -eq 1 ]; then echo "System-wide record [Failed missing output]" err=1 return fi - if ! perf record -aB --synth=no -e cpu-clock,cs --threads=cpu \ - -o "${perfdata}" ${testprog} 2> /dev/null - then - echo "System-wide record [Failed record --threads option]" - err=1 - return - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then - echo "System-wide record [Failed --threads missing output]" + + ret=0 + perf_record_with_retry "${perfdata}" "check_system_wide" "perf test -w thloop" \ + -aB --synth=no -e cpu-clock,cs --threads=cpu || ret=$? + if [ $ret -ne 0 ]; then + echo "System-wide record [Failed record --threads option or missing output]" err=1 return fi echo "Basic --system-wide mode test [Success]" } +check_workload() { + perf report -i "${perfdata}" -q | grep -q "${testsym}" +} + test_workload() { echo "Basic target workload test" - if ! perf record -o "${perfdata}" ${testprog} 2> /dev/null - then - echo "Workload record [Failed record]" + local ret=0 + perf_record_with_retry "${perfdata}" "check_workload" "perf test -w thloop" || ret=$? + if [ $ret -ne 0 ]; then + echo "Workload record [Failed record or missing output]" err=1 return fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then - echo "Workload record [Failed missing output]" - err=1 - return - fi - if ! perf record -e cpu-clock,cs --threads=package \ - -o "${perfdata}" ${testprog} 2> /dev/null - then - echo "Workload record [Failed record --threads option]" - err=1 - return - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then - echo "Workload record [Failed --threads missing output]" + + ret=0 + perf_record_with_retry "${perfdata}" "check_workload" "perf test -w thloop" \ + -e cpu-clock,cs --threads=package || ret=$? + if [ $ret -ne 0 ]; then + echo "Workload record [Failed record --threads option or missing output]" err=1 return fi echo "Basic target workload test [Success]" } +check_branch_counter() { + perf report -i "${perfdata}" -D -q 2>/dev/null | grep -q "$br_cntr_output" && \ + perf script -i "${perfdata}" -F +brstackinsn,+brcntr 2>/dev/null | \ + grep -q "$br_cntr_script_output" +} + test_branch_counter() { echo "Branch counter test" # Check if the branch counter feature is supported @@ -190,67 +198,60 @@ test_branch_counter() { return fi done - if ! perf record -o "${perfdata}" -e "{branches:p,instructions}" -j any,counter ${testprog} 2> /dev/null - then - echo "Branch counter record test [Failed record]" - err=1 - return - fi - if ! perf report -i "${perfdata}" -D -q | grep -q "$br_cntr_output" - then - echo "Branch counter report test [Failed missing output]" - err=1 - return - fi - if ! perf script -i "${perfdata}" -F +brstackinsn,+brcntr | grep -q "$br_cntr_script_output" - then - echo " Branch counter script test [Failed missing output]" + local ret=0 + perf_record_with_retry "${perfdata}" "check_branch_counter" "perf test -w thloop" \ + -e "{branches:p,instructions}" -j any,counter || ret=$? + if [ $ret -ne 0 ]; then + echo "Branch counter test [Failed record or missing output]" err=1 return fi echo "Branch counter test [Success]" } +check_cgroup() { + perf report -i "${perfdata}" -D 2>/dev/null | grep -q "CGROUP" && \ + perf script -i "${perfdata}" -F cgroup 2>/dev/null | grep -q -v "unknown" +} + test_cgroup() { echo "Cgroup sampling test" - if ! perf record -aB --synth=cgroup --all-cgroups -o "${perfdata}" ${testprog} 2> /dev/null - then + local ret=0 + perf_record_with_retry "${perfdata}" "check_cgroup" "perf test -w thloop" \ + -aB --synth=cgroup --all-cgroups || ret=$? + if [ $ret -eq 2 ]; then echo "Cgroup sampling [Skipped not supported]" return - fi - if ! perf report -i "${perfdata}" -D | grep -q "CGROUP" - then + elif [ $ret -eq 1 ]; then echo "Cgroup sampling [Failed missing output]" err=1 return fi - if ! perf script -i "${perfdata}" -F cgroup | grep -q -v "unknown" - then - echo "Cgroup sampling [Failed cannot resolve cgroup names]" - err=1 - return - fi echo "Cgroup sampling test [Success]" } +check_uid() { + perf report -i "${perfdata}" -q | grep -q "${testsym}" +} + test_uid() { echo "Uid sampling test" - if ! perf record -aB --synth=no --uid "$(id -u)" -o "${perfdata}" ${testprog} \ - > "${script_output}" 2>&1 - then - if grep -q "libbpf.*EPERM" "${script_output}" + local ret=0 + perf_record_with_retry "${perfdata}" "check_uid" "perf test -w thloop" \ + -aB --synth=no --uid "$(id -u)" || ret=$? + if [ $ret -eq 2 ]; then + local logfile="${PERF_RECORD_LOGS[${#PERF_RECORD_LOGS[@]}-1]}" + if grep -q -E "libbpf.*EPERM|Access to performance monitoring" "$logfile" || \ + grep -q -E "Permission denied|Failure to open any events" "$logfile" then echo "Uid sampling [Skipped permissions]" return else echo "Uid sampling [Failed to record]" err=1 - # cat "${script_output}" return fi - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then + elif [ $ret -eq 1 ]; then echo "Uid sampling [Failed missing output]" err=1 return diff --git a/tools/perf/tests/shell/record_lbr.sh b/tools/perf/tests/shell/record_lbr.sh index 78a02e90ece1e6..8d51afeb437ba0 100755 --- a/tools/perf/tests/shell/record_lbr.sh +++ b/tools/perf/tests/shell/record_lbr.sh @@ -1,9 +1,12 @@ #!/bin/bash -# perf record LBR tests (exclusive) # SPDX-License-Identifier: GPL-2.0 +# perf record LBR tests set -e +shelldir=$(dirname "$0") +. "${shelldir}"/lib/perf_record.sh + ParanoidAndNotRoot() { [ "$(id -u)" != 0 ] && [ "$(cat /proc/sys/kernel/perf_event_paranoid)" -gt $1 ] } @@ -22,6 +25,7 @@ cleanup() { rm -rf "${perfdata}" rm -rf "${perfdata}".old rm -rf "${perfdata}".txt + perf_record_cleanup trap - EXIT TERM INT } @@ -34,22 +38,28 @@ trap_cleanup() { trap trap_cleanup EXIT TERM INT +check_lbr_callgraph() { + perf report --stitch-lbr -i "${perfdata}" > "${perfdata}".txt 2>&1 +} + lbr_callgraph_test() { test="LBR callgraph" echo "$test" - if ! perf record -e cycles --call-graph lbr -o "${perfdata}" perf test -w thloop - then + set +e + perf_record_with_retry "${perfdata}" "check_lbr_callgraph" "perf test -w thloop" \ + -e cycles --call-graph lbr + local ret=$? + set -e + + if [ $ret -eq 2 ]; then echo "$test [Failed support missing]" if [ $err -eq 0 ] then err=2 fi return - fi - - if ! perf report --stitch-lbr -i "${perfdata}" > "${perfdata}".txt - then + elif [ $ret -eq 1 ]; then cat "${perfdata}".txt echo "$test [Failed in perf report]" err=1 @@ -59,6 +69,12 @@ lbr_callgraph_test() { echo "$test [Success]" } +check_lbr_samples() { + local out + out=$(perf report -D -i "${perfdata}" 2> /dev/null | grep -A1 'PERF_RECORD_SAMPLE') + [ "$(echo "$out" | grep -c 'PERF_RECORD_SAMPLE' || true)" -gt 0 ] +} + lbr_test() { local branch_flags=$1 local test="LBR $2 test" @@ -70,25 +86,27 @@ lbr_test() { local r echo "$test" - if ! perf record -e cycles $branch_flags -o "${perfdata}" perf test -w thloop - then + set +e + perf_record_with_retry "${perfdata}" "check_lbr_samples" "perf test -w thloop" \ + -e cycles $branch_flags + local ret=$? + set -e + + if [ $ret -eq 2 ]; then echo "$test [Failed support missing]" - perf record -e cycles $branch_flags -o "${perfdata}" perf test -w thloop || true if [ $err -eq 0 ] then err=2 fi return - fi - - out=$(perf report -D -i "${perfdata}" 2> /dev/null | grep -A1 'PERF_RECORD_SAMPLE') - sam_nr=$(echo "$out" | grep -c 'PERF_RECORD_SAMPLE' || true) - if [ $sam_nr -eq 0 ] - then + elif [ $ret -eq 1 ]; then echo "$test [Failed no samples captured]" err=1 return fi + + out=$(perf report -D -i "${perfdata}" 2> /dev/null | grep -A1 'PERF_RECORD_SAMPLE') + sam_nr=$(echo "$out" | grep -c 'PERF_RECORD_SAMPLE' || true) echo "$test: $sam_nr samples" bs_nr=$(echo "$out" | grep -c 'branch stack: nr:' || true) diff --git a/tools/perf/tests/shell/stat_all_metrics.sh b/tools/perf/tests/shell/stat_all_metrics.sh index b582d23f28c9e8..feeb34c6fa6df8 100755 --- a/tools/perf/tests/shell/stat_all_metrics.sh +++ b/tools/perf/tests/shell/stat_all_metrics.sh @@ -12,38 +12,65 @@ system_wide_flag="-a" if ParanoidAndNotRoot 0 then system_wide_flag="" - test_prog="perf test -w noploop" + test_prog="perf test -w noploop 0.01" fi +check_metric() { + local output="$1" + local status="$2" + local metric="$3" + + if [[ $status -ne 0 || ! "$output" =~ ${metric:0:50} ]]; then + return 1 + fi + + if [[ "$output" =~ "" || "$output" =~ "" ]]; then + return 1 + fi + + return 0 +} + skip=0 err=3 for m in $(perf list --raw-dump metrics); do echo "Testing $m" result=$(perf stat -M "$m" $system_wide_flag -- $test_prog 2>&1) result_err=$? - if [[ $result_err -eq 0 && "$result" =~ ${m:0:50} ]] - then - # No error result and metric shown. + + if check_metric "$result" $result_err "$m"; then if [[ "$err" -ne 1 ]] then err=0 fi continue fi - if [[ "$result" =~ "Cannot resolve IDs for" || "$result" =~ "No supported events found" ]] + + if [[ "$result" =~ "Access to performance monitoring and observability operations is limited" || \ + "$result" =~ "in per-thread mode, enable system wide" || \ + "$result" =~ "" || \ + "$result" =~ "Cannot resolve IDs for" || \ + "$result" =~ "No supported events found" || \ + "$result" =~ "FP_ARITH" || \ + "$result" =~ "AMX" || \ + "$result" =~ "PMM" ]] then - if [[ $(perf list --raw-dump $m) == "Default"* ]] - then - echo "[Ignored $m] failed but as a Default metric this can be expected" - echo $result + true + else + result=$(perf stat -M "$m" $system_wide_flag -- perf test -w noploop 0.1 2>&1) + result_err=$? + + if check_metric "$result" $result_err "$m"; then + if [[ "$err" -ne 1 ]] + then + err=0 + fi continue fi - echo "[Failed $m] Metric contains missing events" - echo $result - err=1 # Fail - continue - elif [[ "$result" =~ \ - "Access to performance monitoring and observability operations is limited" ]] + fi + + # If retry also failed, determine if we skip, ignore, or fail + if [[ "$result" =~ "Access to performance monitoring and observability operations is limited" ]] then echo "[Skipped $m] Permission failure" echo $result @@ -61,7 +88,9 @@ for m in $(perf list --raw-dump metrics); do skip=1 fi continue - elif [[ "$result" =~ "" ]] + elif [[ "$result" =~ "" || \ + "$result" =~ "Cannot resolve IDs for" || \ + "$result" =~ "No supported events found" ]] then if [[ $(perf list --raw-dump $m) == "Default"* ]] then @@ -105,19 +134,7 @@ for m in $(perf list --raw-dump metrics); do continue fi - # Failed, possibly the workload was too small so retry with something longer. - result=$(perf stat -M "$m" $system_wide_flag -- perf bench internals synthesize 2>&1) - result_err=$? - if [[ $result_err -eq 0 && "$result" =~ ${m:0:50} ]] - then - # No error result and metric shown. - if [[ "$err" -ne 1 ]] - then - err=0 - fi - continue - fi - echo "[Failed $m] has non-zero error '$result_err' or not printed in:" + echo "[Failed $m] has non-zero error '$result_err' or not printed/counted in:" echo "$result" err=1 done diff --git a/tools/perf/tests/shell/stat_bpf_counters.sh b/tools/perf/tests/shell/stat_bpf_counters.sh index 35463358b273ce..11de77ee38ad4e 100755 --- a/tools/perf/tests/shell/stat_bpf_counters.sh +++ b/tools/perf/tests/shell/stat_bpf_counters.sh @@ -4,21 +4,26 @@ set -e -workload="perf test -w sqrtloop" +# Get the first allowed CPU +CPU=$(taskset -c -p $$ | awk -F': ' '{print $2}' | awk -F'[,-]' '{print $1}') +if [ -z "$CPU" ]; then + CPU=0 +fi +workload=(taskset -c "$CPU" awk 'BEGIN { for (i=0; i<10000000; i++) sum+=i }') -# check whether $2 is within +/- 20% of $1 +# check whether $2 is within +/- 15% of $1 compare_number() { first_num=$1 second_num=$2 - # upper bound is first_num * 120% - upper=$(expr $first_num + $first_num / 5 ) - # lower bound is first_num * 80% - lower=$(expr $first_num - $first_num / 5 ) + # upper bound is first_num * 115% + upper=$(expr $first_num + $first_num / 20 \* 3 ) + # lower bound is first_num * 85% + lower=$(expr $first_num - $first_num / 20 \* 3 ) if [ $second_num -gt $upper ] || [ $second_num -lt $lower ]; then - echo "The difference between $first_num and $second_num are greater than 20%." + echo "The difference between $first_num and $second_num are greater than 15%." exit 1 fi } @@ -41,11 +46,12 @@ check_counts() test_bpf_counters() { printf "Testing --bpf-counters " - base_instructions=$(perf stat --no-big-num -e instructions -- $workload 2>&1 | \ + base_instructions=$(perf stat --no-big-num -e instructions:u -- "${workload[@]}" 2>&1 | \ awk -v i=0 -v c=0 '/instructions/ { \ if ($1 != " 0) printf "%.0f", c; else print "&1 | \ + bpf_instructions=$(perf stat --no-big-num --bpf-counters -e instructions:u \ + -- "${workload[@]}" 2>&1 | \ awk -v i=0 -v c=0 '/instructions/ { \ if ($1 != " 0) printf "%.0f", c; else print "&1) + stat_output=$(perf stat --no-big-num \ + -e instructions/name=base_instructions/u,instructions/name=bpf_instructions/bu \ + -- "${workload[@]}" 2>&1) base_instructions=$(echo "$stat_output"| \ awk -v i=0 -v c=0 '/base_instructions/ { \ if ($1 != "/dev/null 2>&1; then + echo "Skipping: no permission to record system-wide events (-a)" + exit 2 +fi + + pythonvalidator=$(dirname $0)/lib/perf_metric_validation.py rulefile=$(dirname $0)/lib/perf_metric_validation_rules.json tmpdir=$(mktemp -d /tmp/__perf_test.program.XXXXX) diff --git a/tools/perf/tests/shell/test_brstack.sh b/tools/perf/tests/shell/test_brstack.sh index eb5837f82e3902..71550e0b37baa4 100755 --- a/tools/perf/tests/shell/test_brstack.sh +++ b/tools/perf/tests/shell/test_brstack.sh @@ -110,20 +110,29 @@ test_trap_eret_branches() { return fi start_err=$err - err=0 - perf record -o $TMPDIR/perf.data --branch-filter any,save_type,u,k -- \ - perf test -w traploop 1000 > "$TMPDIR/record.txt" 2>&1 - perf script -i $TMPDIR/perf.data --fields brstacksym | \ - tr ' ' '\n' > $TMPDIR/perf.script - - # BRBINF.TYPE == TRAP are mapped to PERF_BR_IRQ by the BRBE driver - check_branches "^trap_bench\+[^ ]+/[^ ]/IRQ/" - check_branches "^[^ ]+/trap_bench\+[^ ]+/ERET/" - if [ $err -eq 0 ]; then + local ret=1 + for loops in 1000 10000 100000; do + err=0 + perf record -o $TMPDIR/perf.data --branch-filter any,save_type,u,k -- \ + perf test -w traploop $loops > "$TMPDIR/record.txt" 2>&1 + perf script -i $TMPDIR/perf.data --fields brstacksym | \ + tr ' ' '\n' > $TMPDIR/perf.script + + # BRBINF.TYPE == TRAP are mapped to PERF_BR_IRQ by the BRBE driver + check_branches "^trap_bench\+[^ ]+/[^ ]/IRQ/" + check_branches "^[^ ]+/trap_bench\+[^ ]+/ERET/" + if [ $err -eq 0 ]; then + ret=0 + break + fi + done + + if [ $ret -eq 0 ]; then echo "Testing trap & eret branches [Passed]" err=$start_err else echo "Testing trap & eret branches [Failed]" + err=1 fi } @@ -135,32 +144,40 @@ test_kernel_branches() { return fi start_err=$err - err=0 - perf record -o $TMPDIR/perf.data --branch-filter any,k -- \ - perf bench syscall basic --loop 1000 > "$TMPDIR/record.txt" 2>&1 - perf script -i $TMPDIR/perf.data --fields brstack | \ - tr ' ' '\n' > $TMPDIR/perf.script - - # Example of branch entries: - # "0xffffffff93bda241/0xffffffff93bda20f/M/-/-/..." - # Source addresses come first in user or kernel code. Next is the target - # address that must be in the kernel. - - # Look for source addresses with top bit set - if ! grep -q -E -m1 "^0x[89a-f][0-9a-f]{15}" $TMPDIR/perf.script; then - echo "Testing kernel branch sampling [Failed kernel branches missing]" - err=1 - fi - # Look for no target addresses without top bit set - if grep -q -E -m1 "^0x[0-9a-f]{0,16}/0x[0-7][0-9a-f]{1,15}/" $TMPDIR/perf.script; then - echo "Testing kernel branch sampling [Failed user branches found]" - err=1 - fi - if [ $err -eq 0 ]; then + local ret=1 + for loops in 1000 10000 100000; do + err=0 + perf record -o $TMPDIR/perf.data --branch-filter any,k -- \ + perf bench syscall basic --loop $loops > "$TMPDIR/record.txt" 2>&1 + perf script -i $TMPDIR/perf.data --fields brstack | \ + tr ' ' '\n' > $TMPDIR/perf.script + + # Example of branch entries: + # "0xffffffff93bda241/0xffffffff93bda20f/M/-/-/..." + # Source addresses come first in user or kernel code. Next is the target + # address that must be in the kernel. + + # Look for source addresses with top bit set + if ! grep -q -E -m1 "^0x[89a-f][0-9a-f]{15}" $TMPDIR/perf.script; then + err=1 + fi + # Look for no target addresses without top bit set + if grep -q -E -m1 "^0x[0-9a-f]{0,16}/0x[0-7][0-9a-f]{1,15}/" \ + $TMPDIR/perf.script; then + err=1 + fi + if [ $err -eq 0 ]; then + ret=0 + break + fi + done + + if [ $ret -eq 0 ]; then echo "Testing kernel branch sampling [Passed]" err=$start_err else echo "Testing kernel branch sampling [Failed]" + err=1 fi } @@ -206,20 +223,28 @@ test_syscall() { return fi start_err=$err - err=0 - perf record -o $TMPDIR/perf.data --branch-filter \ - any_call,save_type,u,k -c 10007 -- \ - perf bench syscall basic --loop 8000 > "$TMPDIR/record.txt" 2>&1 - perf script -i $TMPDIR/perf.data --fields brstacksym | \ - tr ' ' '\n' > $TMPDIR/perf.script - - check_branches "getppid[^ ]*/SYSCALL/" + local ret=1 + for loops in 8000 30000 100000; do + err=0 + perf record -o $TMPDIR/perf.data --branch-filter \ + any_call,save_type,u,k -c 10007 -- \ + perf bench syscall basic --loop $loops > "$TMPDIR/record.txt" 2>&1 + perf script -i $TMPDIR/perf.data --fields brstacksym | \ + tr ' ' '\n' > $TMPDIR/perf.script + + check_branches "getppid[^ ]*/SYSCALL/" + if [ $err -eq 0 ]; then + ret=0 + break + fi + done - if [ $err -eq 0 ]; then + if [ $ret -eq 0 ]; then echo "Testing syscalls [Passed]" err=$start_err else echo "Testing syscalls [Failed]" + err=1 fi } set -e diff --git a/tools/perf/tests/shell/test_dwarf_unwind_multithreaded.sh b/tools/perf/tests/shell/test_dwarf_unwind_multithreaded.sh new file mode 100755 index 00000000000000..49e6e3af771f43 --- /dev/null +++ b/tools/perf/tests/shell/test_dwarf_unwind_multithreaded.sh @@ -0,0 +1,65 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Test libdw unwinding of multi-threaded processes (exclusive) + +set -e + +if ! perf check feature -q libdw-dwarf-unwind; then + echo "Skip: libdw DWARF unwinding is not available" + exit 2 +fi + +tmpdir=$(mktemp -d /tmp/perf-test-dwarf-unwind-multithreaded.XXXXXX) +perf_data="$tmpdir/perf.data" +perf_script="$tmpdir/perf-script.txt" +nr_threads=4 +nr_worker_threads=$((nr_threads - 1)) + +cleanup() +{ + trap - EXIT TERM INT + rm -rf "$tmpdir" +} + +trap cleanup EXIT TERM INT + +if ! perf record -q -e task-clock:u -F 99 --call-graph dwarf,8192 \ + -o "$perf_data" -- perf test -w thloop 2 "$nr_threads" +then + echo "Skip: failed to record task-clock:u" + exit 2 +fi + +if ! perf script --unwind-style=libdw \ + -F comm,pid,tid,event,ip,sym -i "$perf_data" > "$perf_script" +then + echo "Error: failed to process the recording with libdw" >&2 + exit 1 +fi + +nr_unwound_threads=$( + awk ' + BEGIN { RS = "" } + + # thfunc is the worker-only caller of test_loop. Finding it proves + # that libdw unwound beyond the sampled leaf for this worker TID. + /thfunc/ { + split($2, id, "/") + seen[id[2]] = 1 + } + + END { + for (tid in seen) + nr_tids++ + print nr_tids + 0 + } + ' "$perf_script" +) + +if [ "$nr_unwound_threads" -ne "$nr_worker_threads" ]; then + echo "Error: expected callchains for $nr_worker_threads worker TIDs," \ + "found $nr_unwound_threads" >&2 + exit 1 +fi + +exit 0 diff --git a/tools/perf/tests/shell/trace_record_replay.sh b/tools/perf/tests/shell/trace_record_replay.sh index 88d30a03dcecb0..38fcafcdfb91cc 100755 --- a/tools/perf/tests/shell/trace_record_replay.sh +++ b/tools/perf/tests/shell/trace_record_replay.sh @@ -6,16 +6,46 @@ # shellcheck source=lib/probe.sh . "$(dirname $0)"/lib/probe.sh +# shellcheck source=lib/perf_record.sh +. "$(dirname $0)"/lib/perf_record.sh skip_if_no_perf_trace || exit 2 [ "$(id -u)" = 0 ] || exit 2 file=$(mktemp /tmp/temporary_file.XXXXX) +err=0 -perf trace record -o ${file} sleep 1 || exit 1 -if ! perf trace -i ${file} 2>&1 | grep nanosleep; then - echo "Failed: cannot find *nanosleep syscall" +cleanup() { + rm -f ${file} + perf_record_cleanup + trap - EXIT INT TERM +} + +trap_cleanup() { + echo "Unexpected signal in ${FUNCNAME[1]}" + cleanup + exit 1 +} +trap trap_cleanup EXIT INT TERM + +check_nanosleep() { + perf trace -i "${file}" 2>&1 | grep -q nanosleep +} + +PERF_RECORD_CMD="perf trace record" perf_record_with_retry "${file}" "check_nanosleep" "sleep" +err=$? + +if [ $err -ne 0 ]; then + if [ $err -eq 2 ]; then + logfile="${PERF_RECORD_LOGS[${#PERF_RECORD_LOGS[@]}-1]}" + echo "perf trace record failed. Log output:" + cat "$logfile" + else + echo "Failed: cannot find *nanosleep syscall" + fi + cleanup exit 1 fi -rm -f ${file} +cleanup +exit 0 diff --git a/tools/perf/tests/sw-clock.c b/tools/perf/tests/sw-clock.c index b6e46975379cd5..d181858816352c 100644 --- a/tools/perf/tests/sw-clock.c +++ b/tools/perf/tests/sw-clock.c @@ -59,7 +59,7 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id) evsel = evsel__new(&attr); if (evsel == NULL) { pr_debug("evsel__new\n"); - goto out_delete_evlist; + goto out_put_evlist; } evlist__add(evlist, evsel); @@ -68,10 +68,10 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id) if (!cpus || !threads) { err = -ENOMEM; pr_debug("Not enough memory to create thread/cpu maps\n"); - goto out_delete_evlist; + goto out_put_evlist; } - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); if (evlist__open(evlist)) { const char *knob = "/proc/sys/kernel/perf_event_max_sample_rate"; @@ -80,14 +80,14 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id) pr_debug("Couldn't open evlist: %s\nHint: check %s, using %" PRIu64 " in this test.\n", str_error_r(errno, sbuf, sizeof(sbuf)), knob, (u64)attr.sample_freq); - goto out_delete_evlist; + goto out_put_evlist; } - err = evlist__mmap(evlist, 128); + err = evlist__do_mmap(evlist, 128); if (err < 0) { pr_debug("failed to mmap event: %d (%s)\n", errno, str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } evlist__enable(evlist); @@ -98,7 +98,7 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id) evlist__disable(evlist); - md = &evlist->mmap[0]; + md = &evlist__mmap(evlist)[0]; if (perf_mmap__read_init(&md->core) < 0) goto out_init; @@ -113,7 +113,7 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id) if (err < 0) { pr_debug("Error during parse sample\n"); perf_sample__exit(&sample); - goto out_delete_evlist; + goto out_put_evlist; } total_periods += sample.period; @@ -131,10 +131,10 @@ out_init: err = -1; } -out_delete_evlist: +out_put_evlist: perf_cpu_map__put(cpus); perf_thread_map__put(threads); - evlist__delete(evlist); + evlist__put(evlist); return err; } diff --git a/tools/perf/tests/switch-tracking.c b/tools/perf/tests/switch-tracking.c index e32331fee2778c..73568c782d72b3 100644 --- a/tools/perf/tests/switch-tracking.c +++ b/tools/perf/tests/switch-tracking.c @@ -237,6 +237,7 @@ static int add_event(struct evlist *evlist, struct list_head *events, if (evlist__parse_sample(evlist, event, &sample)) { pr_debug("evlist__parse_sample failed\n"); + perf_sample__exit(&sample); return -1; } @@ -282,8 +283,8 @@ static int process_events(struct evlist *evlist, struct mmap *md; int i, ret; - for (i = 0; i < evlist->core.nr_mmaps; i++) { - md = &evlist->mmap[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; @@ -374,7 +375,7 @@ static int test__switch_tracking(struct test_suite *test __maybe_unused, int sub goto out_err; } - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); /* First event */ err = parse_event(evlist, "cpu-clock:u"); @@ -471,7 +472,7 @@ static int test__switch_tracking(struct test_suite *test __maybe_unused, int sub goto out; } - err = evlist__mmap(evlist, UINT_MAX); + err = evlist__do_mmap(evlist, UINT_MAX); if (err) { pr_debug("evlist__mmap failed!\n"); goto out_err; @@ -582,7 +583,7 @@ static int test__switch_tracking(struct test_suite *test __maybe_unused, int sub out: if (evlist) { evlist__disable(evlist); - evlist__delete(evlist); + evlist__put(evlist); } perf_cpu_map__put(cpus); perf_thread_map__put(threads); diff --git a/tools/perf/tests/task-exit.c b/tools/perf/tests/task-exit.c index 4053ff2813bb79..95393edbfe363e 100644 --- a/tools/perf/tests/task-exit.c +++ b/tools/perf/tests/task-exit.c @@ -74,15 +74,15 @@ static int test__task_exit(struct test_suite *test __maybe_unused, int subtest _ if (!cpus || !threads) { err = -ENOMEM; pr_debug("Not enough memory to create thread/cpu maps\n"); - goto out_delete_evlist; + goto out_put_evlist; } - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); err = evlist__prepare_workload(evlist, &target, argv, false, workload_exec_failed_signal); if (err < 0) { pr_debug("Couldn't run the workload!\n"); - goto out_delete_evlist; + goto out_put_evlist; } evsel = evlist__first(evlist); @@ -101,20 +101,20 @@ static int test__task_exit(struct test_suite *test __maybe_unused, int subtest _ if (err < 0) { pr_debug("Couldn't open the evlist: %s\n", str_error_r(-err, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } - if (evlist__mmap(evlist, 128) < 0) { + if (evlist__do_mmap(evlist, 128) < 0) { pr_debug("failed to mmap events: %d (%s)\n", errno, str_error_r(errno, sbuf, sizeof(sbuf))); err = -1; - goto out_delete_evlist; + goto out_put_evlist; } evlist__start_workload(evlist); retry: - md = &evlist->mmap[0]; + md = &evlist__mmap(evlist)[0]; if (perf_mmap__read_init(&md->core) < 0) goto out_init; @@ -133,7 +133,7 @@ out_init: if (retry_count++ > 1000) { pr_debug("Failed after retrying 1000 times\n"); err = -1; - goto out_delete_evlist; + goto out_put_evlist; } goto retry; @@ -144,10 +144,10 @@ out_init: err = -1; } -out_delete_evlist: +out_put_evlist: perf_cpu_map__put(cpus); perf_thread_map__put(threads); - evlist__delete(evlist); + evlist__put(evlist); return err; } diff --git a/tools/perf/tests/time-utils-test.c b/tools/perf/tests/time-utils-test.c index 38df10373c1e17..90a9a4b4f17897 100644 --- a/tools/perf/tests/time-utils-test.c +++ b/tools/perf/tests/time-utils-test.c @@ -69,16 +69,19 @@ struct test_data { static bool test__perf_time__parse_for_ranges(struct test_data *d) { - struct evlist evlist = { - .first_sample_time = d->first, - .last_sample_time = d->last, - }; - struct perf_session session = { .evlist = &evlist }; + struct evlist *evlist = evlist__new(); + struct perf_session session = { .evlist = evlist }; struct perf_time_interval *ptime = NULL; int range_size, range_num; bool pass = false; int i, err; + if (!evlist) { + pr_debug("Missing evlist\n"); + return false; + } + evlist__set_first_sample_time(evlist, d->first); + evlist__set_last_sample_time(evlist, d->last); pr_debug("\nperf_time__parse_for_ranges(\"%s\")\n", d->str); if (strchr(d->str, '%')) @@ -127,6 +130,7 @@ static bool test__perf_time__parse_for_ranges(struct test_data *d) pass = true; out: + evlist__put(evlist); free(ptime); return pass; } diff --git a/tools/perf/tests/tool_pmu.c b/tools/perf/tests/tool_pmu.c index 1e900ef92e3780..c6c5ebf0e935f0 100644 --- a/tools/perf/tests/tool_pmu.c +++ b/tools/perf/tests/tool_pmu.c @@ -40,9 +40,10 @@ static int do_test(enum tool_pmu_event ev, bool with_pmu) } ret = TEST_OK; - if (with_pmu ? (evlist->core.nr_entries != 1) : (evlist->core.nr_entries < 1)) { + if (with_pmu ? (evlist__nr_entries(evlist) != 1) + : (evlist__nr_entries(evlist) < 1)) { pr_debug("FAILED %s:%d Unexpected number of events for '%s' of %d\n", - __FILE__, __LINE__, str, evlist->core.nr_entries); + __FILE__, __LINE__, str, evlist__nr_entries(evlist)); ret = TEST_FAIL; goto out; } @@ -67,7 +68,7 @@ static int do_test(enum tool_pmu_event ev, bool with_pmu) out: parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); return ret; } diff --git a/tools/perf/tests/topology.c b/tools/perf/tests/topology.c index bd7b859dea66a6..77cb8318c0b18b 100644 --- a/tools/perf/tests/topology.c +++ b/tools/perf/tests/topology.c @@ -46,7 +46,7 @@ static int session_write_header(char *path) session->evlist = evlist__new_default(&target, /*sample_callchains=*/false); TEST_ASSERT_VAL("can't get evlist", session->evlist); - session->evlist->session = session; + evlist__set_session(session->evlist, session); perf_header__set_feat(&session->header, HEADER_CPU_TOPOLOGY); perf_header__set_feat(&session->header, HEADER_NRCPUS); @@ -58,7 +58,7 @@ static int session_write_header(char *path) !perf_session__write_header(session, session->evlist, perf_data__fd(&data), true)); - evlist__delete(session->evlist); + evlist__put(session->evlist); perf_session__delete(session); return 0; diff --git a/tools/perf/tests/uncore-event-sorting.c b/tools/perf/tests/uncore-event-sorting.c index 7d2fc304e21fec..7756777c54c2e5 100644 --- a/tools/perf/tests/uncore-event-sorting.c +++ b/tools/perf/tests/uncore-event-sorting.c @@ -147,8 +147,8 @@ static int test__uncore_event_sorting(struct test_suite *test __maybe_unused, goto out_err; } - CHECK_COND(evlist->core.nr_entries >= 4, "Number of events is >= 4"); - CHECK_EQUAL(evlist->core.nr_entries % 2, 0, "Number of events is a multiple of 2"); + CHECK_COND(evlist__nr_entries(evlist) >= 4, "Number of events is >= 4"); + CHECK_EQUAL(evlist__nr_entries(evlist) % 2, 0, "Number of events is a multiple of 2"); evlist__for_each_entry(evlist, evsel) { struct evsel *next; @@ -165,7 +165,7 @@ static int test__uncore_event_sorting(struct test_suite *test __maybe_unused, ret = TEST_OK; out_err: - evlist__delete(evlist); + evlist__put(evlist); parse_events_error__exit(&err); zfree(&pmu_prefix); zfree(&m.event1); diff --git a/tools/perf/ui/browsers/annotate.c b/tools/perf/ui/browsers/annotate.c index 97ae4c86bebbe3..e47a467750890d 100644 --- a/tools/perf/ui/browsers/annotate.c +++ b/tools/perf/ui/browsers/annotate.c @@ -597,7 +597,7 @@ static bool annotate_browser__callq(struct annotate_browser *browser, notes = symbol__annotation(dl->ops.target.sym); annotation__lock(notes); - if (!symbol__hists(dl->ops.target.sym, evsel->evlist->core.nr_entries)) { + if (!symbol__hists(dl->ops.target.sym, evlist__nr_entries(evsel->evlist))) { annotation__unlock(notes); ui__warning("Not enough memory for annotating '%s' symbol!\n", dl->ops.target.sym->name); @@ -1201,7 +1201,7 @@ int __hist_entry__tui_annotate(struct hist_entry *he, struct map_symbol *ms, ui__warning("Annotation has no source code."); } } else { - err = thread__get_arch(ms->thread, &browser.arch); + err = map_symbol__get_arch(ms, &browser.arch); if (err) { annotate_browser__symbol_annotate_error(&browser, err); return -1; diff --git a/tools/perf/ui/browsers/hists.c b/tools/perf/ui/browsers/hists.c index cfa6386e6e1daf..96fd598dbb6d53 100644 --- a/tools/perf/ui/browsers/hists.c +++ b/tools/perf/ui/browsers/hists.c @@ -688,10 +688,10 @@ static int hist_browser__handle_hotkey(struct hist_browser *browser, bool warn_l ui_browser__update_nr_entries(&browser->b, nr_entries); if (warn_lost_event && - (evsel->evlist->stats.nr_lost_warned != - evsel->evlist->stats.nr_events[PERF_RECORD_LOST])) { - evsel->evlist->stats.nr_lost_warned = - evsel->evlist->stats.nr_events[PERF_RECORD_LOST]; + (evlist__stats(evsel->evlist)->nr_lost_warned != + evlist__stats(evsel->evlist)->nr_events[PERF_RECORD_LOST])) { + evlist__stats(evsel->evlist)->nr_lost_warned = + evlist__stats(evsel->evlist)->nr_events[PERF_RECORD_LOST]; ui_browser__warn_lost_events(&browser->b); } @@ -2356,6 +2356,16 @@ static int hists_browser__scnprintf_title(struct hist_browser *browser, char *bf return printed; } +struct popup_action { + unsigned long time; + struct thread *thread; + int (*fn)(struct hist_browser *browser, struct popup_action *act); + struct map_symbol ms; + int socket; + enum rstype rstype; + +}; + static inline void free_popup_options(char **options, int n) { int i; @@ -2364,6 +2374,16 @@ static inline void free_popup_options(char **options, int n) zfree(&options[i]); } +static inline void free_popup_actions(struct popup_action *actions, int n) +{ + int i; + + for (i = 0; i < n; ++i) { + map_symbol__exit(&actions[i].ms); + memset(&actions[i], 0, sizeof(struct popup_action)); + } +} + /* * Only runtime switching of perf data file will make "input_name" point * to a malloced buffer. So add "is_input_name_malloced" flag to decide @@ -2453,16 +2473,6 @@ close_file_and_continue: return ret; } -struct popup_action { - unsigned long time; - struct thread *thread; - int (*fn)(struct hist_browser *browser, struct popup_action *act); - struct map_symbol ms; - int socket; - enum rstype rstype; - -}; - static int do_annotate(struct hist_browser *browser, struct popup_action *act) { @@ -2540,7 +2550,7 @@ add_annotate_opt(struct popup_action *act, char **optstr, if (asprintf(optstr, "Annotate %s", ms->sym->name) < 0) return 0; - act->ms = *ms; + map_symbol__copy(&act->ms, ms); act->fn = do_annotate; return 1; } @@ -2572,7 +2582,7 @@ add_annotate_type_opt(struct popup_action *act, char **optstr, static int do_zoom_thread(struct hist_browser *browser, struct popup_action *act) { - struct thread *thread = act->thread; + struct thread *thread = act->ms.thread; if ((!hists__has(browser->hists, thread) && !hists__has(browser->hists, comm)) || thread == NULL) @@ -2627,7 +2637,7 @@ add_thread_opt(struct hist_browser *browser, struct popup_action *act, if (ret < 0) return 0; - act->thread = thread; + act->ms.thread = thread__get(thread); act->fn = do_zoom_thread; return 1; } @@ -2640,13 +2650,14 @@ static int hists_browser__zoom_map(struct hist_browser *browser, struct map *map if (browser->hists->dso_filter) { pstack__remove(browser->pstack, &browser->hists->dso_filter); perf_hpp__set_elide(HISTC_DSO, false); + dso__put((struct dso *)browser->hists->dso_filter); browser->hists->dso_filter = NULL; ui_helpline__pop(); } else { struct dso *dso = map__dso(map); ui_helpline__fpush("To zoom out press ESC or ENTER + \"Zoom out of %s DSO\"", __map__is_kernel(map) ? "the Kernel" : dso__short_name(dso)); - browser->hists->dso_filter = dso; + browser->hists->dso_filter = dso__get(dso); perf_hpp__set_elide(HISTC_DSO, true); pstack__push(browser->pstack, &browser->hists->dso_filter); } @@ -2674,7 +2685,7 @@ add_dso_opt(struct hist_browser *browser, struct popup_action *act, __map__is_kernel(map) ? "the Kernel" : dso__short_name(map__dso(map))) < 0) return 0; - act->ms.map = map; + act->ms.map = map__get(map); act->fn = do_zoom_dso; return 1; } @@ -2719,7 +2730,7 @@ add_map_opt(struct hist_browser *browser, if (asprintf(optstr, "Browse map details") < 0) return 0; - act->ms.map = map; + act->ms.map = map__get(map); act->fn = do_browse_map; return 1; } @@ -2733,8 +2744,8 @@ do_run_script(struct hist_browser *browser, int n = 0; len = 100; - if (act->thread) - len += strlen(thread__comm_str(act->thread)); + if (act->ms.thread) + len += strlen(thread__comm_str(act->ms.thread)); else if (act->ms.sym) len += strlen(act->ms.sym->name); script_opt = malloc(len); @@ -2742,9 +2753,9 @@ do_run_script(struct hist_browser *browser, return -1; script_opt[0] = 0; - if (act->thread) { + if (act->ms.thread) { n = scnprintf(script_opt, len, " -c %s ", - thread__comm_str(act->thread)); + thread__comm_str(act->ms.thread)); } else if (act->ms.sym) { n = scnprintf(script_opt, len, " -S %s ", act->ms.sym->name); @@ -2799,7 +2810,7 @@ add_script_opt_2(struct popup_action *act, char **optstr, return 0; } - act->thread = thread; + act->ms.thread = thread__get(thread); act->ms.sym = sym; act->fn = do_run_script; return 1; @@ -3003,9 +3014,9 @@ static int evsel__hists_browse(struct evsel *evsel, int nr_events, const char *h struct hists *hists = evsel__hists(evsel); struct hist_browser *browser = perf_evsel_browser__new(evsel, hbt, env); struct branch_info *bi = NULL; -#define MAX_OPTIONS 16 +#define MAX_OPTIONS 32 char *options[MAX_OPTIONS]; - struct popup_action actions[MAX_OPTIONS]; + struct popup_action actions[MAX_OPTIONS], hotkey_act; int nr_options = 0; int key = -1; char buf[128]; @@ -3064,15 +3075,15 @@ static int evsel__hists_browse(struct evsel *evsel, int nr_events, const char *h browser->min_pcnt = min_pcnt; hist_browser__update_nr_entries(browser); + memset(options, 0, sizeof(options)); + memset(actions, 0, sizeof(actions)); + browser->pstack = pstack__new(3); if (browser->pstack == NULL) goto out; ui_helpline__push(helpline); - memset(options, 0, sizeof(options)); - memset(actions, 0, sizeof(actions)); - if (symbol_conf.col_width_list_str) perf_hpp__set_user_width(symbol_conf.col_width_list_str); @@ -3087,6 +3098,8 @@ static int evsel__hists_browse(struct evsel *evsel, int nr_events, const char *h key = 0; // reset key do_hotkey: // key came straight from options ui__popup_menu() + free_popup_options(options, MAX_OPTIONS); + free_popup_actions(actions, MAX_OPTIONS); choice = nr_options = 0; key = hist_browser__run(browser, helpline, warn_lost_event, key); @@ -3146,24 +3159,40 @@ do_hotkey: // key came straight from options ui__popup_menu() } if (!browser->selection->sym) { + struct map_symbol source_ms; + if (!browser->he_selection) continue; + memset(&source_ms, 0, sizeof(source_ms)); + if (sort__mode == SORT_MODE__BRANCH) { bi = browser->he_selection->branch_info; if (!bi || !bi->to.ms.map) continue; - actions->ms.sym = symbol__new_unresolved(bi->to.al_addr, bi->to.ms.map); - actions->ms.map = bi->to.ms.map; + source_ms.sym = + symbol__new_unresolved( + bi->to.al_addr, + bi->to.ms.map); + source_ms.thread = bi->to.ms.thread; + source_ms.map = bi->to.ms.map; } else { - actions->ms.sym = symbol__new_unresolved(browser->he_selection->ip, - browser->selection->map); - actions->ms.map = browser->selection->map; + source_ms.sym = + symbol__new_unresolved( + browser->he_selection->ip, + browser->selection->map); + source_ms.thread = browser->selection->thread; + source_ms.map = browser->selection->map; } - if (!actions->ms.sym) + if (!source_ms.sym) continue; + + memset(&hotkey_act, 0, sizeof(hotkey_act)); + map_symbol__copy(&hotkey_act.ms, &source_ms); + do_annotate(browser, &hotkey_act); + map_symbol__exit(&hotkey_act.ms); } else { if (symbol__annotation(browser->selection->sym)->src == NULL) { ui_browser__warning(&browser->b, delay_secs * 2, @@ -3173,18 +3202,20 @@ do_hotkey: // key came straight from options ui__popup_menu() continue; } - actions->ms.map = browser->selection->map; - actions->ms.sym = browser->selection->sym; + memset(&hotkey_act, 0, sizeof(hotkey_act)); + map_symbol__copy(&hotkey_act.ms, browser->selection); + do_annotate(browser, &hotkey_act); + map_symbol__exit(&hotkey_act.ms); } - - do_annotate(browser, actions); continue; case 'P': hist_browser__dump(browser); continue; case 'd': - actions->ms.map = map; - do_zoom_dso(browser, actions); + memset(&hotkey_act, 0, sizeof(hotkey_act)); + hotkey_act.ms.map = map__get(map); + do_zoom_dso(browser, &hotkey_act); + map_symbol__exit(&hotkey_act.ms); continue; case 'k': if (browser->selection != NULL) @@ -3199,12 +3230,16 @@ do_hotkey: // key came straight from options ui__popup_menu() verbose); continue; case 't': - actions->thread = thread; - do_zoom_thread(browser, actions); + memset(&hotkey_act, 0, sizeof(hotkey_act)); + hotkey_act.ms.thread = thread__get(thread); + do_zoom_thread(browser, &hotkey_act); + map_symbol__exit(&hotkey_act.ms); continue; case 'S': - actions->socket = socked_id; - do_zoom_socket(browser, actions); + memset(&hotkey_act, 0, sizeof(hotkey_act)); + hotkey_act.socket = socked_id; + do_zoom_socket(browser, &hotkey_act); + map_symbol__exit(&hotkey_act.ms); continue; case '/': if (ui_browser__input_window("Symbol to show", @@ -3219,9 +3254,11 @@ do_hotkey: // key came straight from options ui__popup_menu() continue; case 'r': if (is_report_browser(hbt)) { - actions->thread = NULL; - actions->ms.sym = NULL; - do_run_script(browser, actions); + memset(&hotkey_act, 0, sizeof(hotkey_act)); + hotkey_act.ms.thread = NULL; + hotkey_act.ms.sym = NULL; + do_run_script(browser, &hotkey_act); + map_symbol__exit(&hotkey_act.ms); } continue; case 's': @@ -3293,20 +3330,19 @@ do_hotkey: // key came straight from options ui__popup_menu() continue; } - actions->ms.map = map; + memset(&hotkey_act, 0, sizeof(hotkey_act)); top = pstack__peek(browser->pstack); if (top == &browser->hists->dso_filter) { - /* - * No need to set actions->dso here since - * it's just to remove the current filter. - */ - do_zoom_dso(browser, actions); + hotkey_act.ms.map = map__get(map); + do_zoom_dso(browser, &hotkey_act); } else if (top == &browser->hists->thread_filter) { - actions->thread = thread; - do_zoom_thread(browser, actions); + hotkey_act.ms.thread = thread__get(thread); + do_zoom_thread(browser, &hotkey_act); } else if (top == &browser->hists->socket_filter) { - do_zoom_socket(browser, actions); + hotkey_act.socket = socked_id; + do_zoom_socket(browser, &hotkey_act); } + map_symbol__exit(&hotkey_act.ms); continue; } case 'q': @@ -3321,7 +3357,7 @@ do_hotkey: // key came straight from options ui__popup_menu() * No need to refresh, resort/decay histogram * entries if we are not collecting samples: */ - if (top->evlist->enabled) { + if (evlist__enabled(top->evlist)) { helpline = "Press 'f' to disable the events or 'h' to see other hotkeys"; hbt->refresh = delay_secs; } else { @@ -3443,9 +3479,20 @@ skip_scripting: if (key == K_SWITCH_INPUT_DATA) break; + + free_popup_options(options, MAX_OPTIONS); + free_popup_actions(actions, MAX_OPTIONS); } out_free_stack: pstack__delete(browser->pstack); + free_popup_actions(actions, MAX_OPTIONS); + thread__zput(hists->thread_filter); + dso__put((struct dso *)hists->dso_filter); + hists->dso_filter = NULL; + perf_hpp__set_elide(HISTC_DSO, false); + perf_hpp__set_elide(HISTC_THREAD, false); + hists__filter_by_dso(hists); + hists__filter_by_thread(hists); out: hist_browser__delete(browser); free_popup_options(options, MAX_OPTIONS); @@ -3493,7 +3540,7 @@ static void perf_evsel_menu__write(struct ui_browser *browser, unit, unit == ' ' ? "" : " ", ev_name); ui_browser__printf(browser, "%s", bf); - nr_events = evsel->evlist->stats.nr_events[PERF_RECORD_LOST]; + nr_events = evlist__stats(evsel->evlist)->nr_events[PERF_RECORD_LOST]; if (nr_events != 0) { menu->lost_events = true; if (!current_entry) @@ -3559,13 +3606,13 @@ browse_hists: ui_browser__show_title(&menu->b, title); switch (key) { case K_TAB: - if (pos->core.node.next == &evlist->core.entries) + if (pos->core.node.next == &evlist__core(evlist)->entries) pos = evlist__first(evlist); else pos = evsel__next(pos); goto browse_hists; case K_UNTAB: - if (pos->core.node.prev == &evlist->core.entries) + if (pos->core.node.prev == &evlist__core(evlist)->entries) pos = evlist__last(evlist); else pos = evsel__prev(pos); @@ -3618,7 +3665,7 @@ static int __evlist__tui_browse_hists(struct evlist *evlist, int nr_entries, con struct evsel *pos; struct evsel_menu menu = { .b = { - .entries = &evlist->core.entries, + .entries = &evlist__core(evlist)->entries, .refresh = ui_browser__list_head_refresh, .seek = ui_browser__list_head_seek, .write = perf_evsel_menu__write, @@ -3646,7 +3693,7 @@ static int __evlist__tui_browse_hists(struct evlist *evlist, int nr_entries, con static bool evlist__single_entry(struct evlist *evlist) { - int nr_entries = evlist->core.nr_entries; + int nr_entries = evlist__nr_entries(evlist); if (nr_entries == 1) return true; @@ -3664,7 +3711,7 @@ static bool evlist__single_entry(struct evlist *evlist) int evlist__tui_browse_hists(struct evlist *evlist, const char *help, struct hist_browser_timer *hbt, float min_pcnt, struct perf_env *env, bool warn_lost_event) { - int nr_entries = evlist->core.nr_entries; + int nr_entries = evlist__nr_entries(evlist); if (evlist__single_entry(evlist)) { single_entry: { diff --git a/tools/perf/util/amd-sample-raw.c b/tools/perf/util/amd-sample-raw.c index 394c061fbeb38d..cda3836329c3a3 100644 --- a/tools/perf/util/amd-sample-raw.c +++ b/tools/perf/util/amd-sample-raw.c @@ -421,7 +421,7 @@ static void parse_cpuid(struct perf_env *env) */ bool evlist__has_amd_ibs(struct evlist *evlist) { - struct perf_env *env = perf_session__env(evlist->session); + struct perf_env *env = perf_session__env(evlist__session(evlist)); int ret, nr_pmu_mappings = perf_env__nr_pmu_mappings(env); const char *pmu_mapping = perf_env__pmu_mappings(env); char name[sizeof("ibs_fetch")]; diff --git a/tools/perf/util/annotate-data.c b/tools/perf/util/annotate-data.c index 63e3c54fab421f..4e4c587640823c 100644 --- a/tools/perf/util/annotate-data.c +++ b/tools/perf/util/annotate-data.c @@ -1829,7 +1829,7 @@ int annotated_data_type__update_samples(struct annotated_data_type *adt, return 0; if (adt->histograms == NULL) { - int nr = evsel->evlist->core.nr_entries; + int nr = evlist__nr_entries(evsel->evlist); if (alloc_data_type_histograms(adt, nr) < 0) return -1; diff --git a/tools/perf/util/annotate.c b/tools/perf/util/annotate.c index 02505222d8c2d4..df70e95a847047 100644 --- a/tools/perf/util/annotate.c +++ b/tools/perf/util/annotate.c @@ -328,7 +328,7 @@ static int symbol__inc_addr_samples(struct map_symbol *ms, if (sym == NULL) return 0; - src = symbol__hists(sym, sample->evsel->evlist->core.nr_entries); + src = symbol__hists(sym, evlist__nr_entries(sample->evsel->evlist)); return src ? __symbol__inc_addr_samples(ms, src, addr, sample) : 0; } @@ -339,7 +339,7 @@ static int symbol__account_br_cntr(struct annotated_branch *branch, { unsigned int br_cntr_nr = evsel__leader(evsel)->br_cntr_nr; unsigned int base = evsel__leader(evsel)->br_cntr_idx; - unsigned int off = offset * evsel->evlist->nr_br_cntr; + unsigned int off = offset * evlist__nr_br_cntr(evsel->evlist); u64 *branch_br_cntr = branch->br_cntr; unsigned int i, mask, width; @@ -369,7 +369,7 @@ static int symbol__account_cycles(u64 addr, u64 start, struct symbol *sym, if (sym == NULL) return 0; - branch = symbol__find_branch_hist(sym, evsel->evlist->nr_br_cntr); + branch = symbol__find_branch_hist(sym, evlist__nr_br_cntr(evsel->evlist)); if (!branch) return -ENOMEM; if (addr < sym->start || addr >= sym->end) @@ -511,7 +511,7 @@ static void annotation__count_and_fill(struct annotation *notes, u64 start, u64 static int annotation__compute_ipc(struct annotation *notes, size_t size, struct evsel *evsel) { - unsigned int br_cntr_nr = evsel->evlist->nr_br_cntr; + unsigned int br_cntr_nr = evlist__nr_br_cntr(evsel->evlist); int err = 0; s64 offset; @@ -982,24 +982,43 @@ void symbol__calc_percent(struct symbol *sym, struct evsel *evsel) annotation__calc_percent(notes, evsel, symbol__size(sym)); } -int thread__get_arch(struct thread *thread, const struct arch **parch) + + +int map_symbol__get_arch(struct map_symbol *ms, const struct arch **parch) { const struct arch *arch; - struct machine *machine; - uint32_t e_flags; - uint16_t e_machine; + struct machine *machine = NULL; + struct map *map = ms->map; + struct dso *dso = map ? map__dso(map) : NULL; + uint32_t e_flags = 0; + uint16_t e_machine = EM_NONE; - if (!thread) { - *parch = NULL; - return -1; + const char *cpuid = NULL; + + if (ms->thread) { + machine = maps__machine(thread__maps(ms->thread)); + e_machine = thread__e_machine(ms->thread, machine, &e_flags); + if (machine && machine->env) + cpuid = machine->env->cpuid; + } else if (dso) { + struct maps *kmaps = (map && dso__kernel(dso)) ? map__kmaps(map) : NULL; + struct machine *kmap_machine = kmaps ? maps__machine(kmaps) : NULL; + + e_machine = dso__e_machine(dso, kmap_machine, &e_flags); + if (kmap_machine && kmap_machine->env) + cpuid = kmap_machine->env->cpuid; } - machine = maps__machine(thread__maps(thread)); - e_machine = thread__e_machine(thread, machine, &e_flags); - arch = arch__find(e_machine, e_flags, machine->env ? machine->env->cpuid : NULL); + if (e_machine == EM_NONE) + e_machine = thread__e_machine(NULL, NULL, &e_flags); + + arch = arch__find(e_machine, e_flags, cpuid); if (arch == NULL) { pr_err("%s: unsupported arch %d\n", __func__, e_machine); - return errno; + /* TODO: Refactor annotate/disassemble subsystem error + * codes to uniformly return negative integers. + */ + return errno ? errno : ENOTSUP; } if (parch) *parch = arch; @@ -1018,7 +1037,7 @@ int symbol__annotate(struct map_symbol *ms, struct evsel *evsel, const struct arch *arch = NULL; int err, nr; - err = thread__get_arch(ms->thread, &arch); + err = map_symbol__get_arch(ms, &arch); if (err) return err; @@ -1251,6 +1270,11 @@ int hist_entry__annotate_printf(struct hist_entry *he, struct evsel *evsel) evsel_name = buf; } + if (map_symbol__get_arch(ms, &apd.arch)) { + free(filename); + return ENOTSUP; + } + graph_dotted_len = printf(" %-*.*s| Source code & Disassembly of %s for %s (%" PRIu64 " samples, " "percent: %s)\n", width, width, symbol_conf.show_total_period ? "Period" : @@ -1266,7 +1290,6 @@ int hist_entry__annotate_printf(struct hist_entry *he, struct evsel *evsel) apd.addr_fmt_width = annotated_source__addr_fmt_width(¬es->src->source, notes->src->start); - thread__get_arch(ms->thread, &apd.arch); apd.dbg = dso__debuginfo(dso); list_for_each_entry(pos, ¬es->src->source, node) { @@ -1371,7 +1394,7 @@ static int symbol__annotate_fprintf2(struct symbol *sym, FILE *fp, struct annotation_line *al; if (annotate_opts.code_with_type) { - thread__get_arch(apd->he->ms.thread, &apd->arch); + map_symbol__get_arch(&apd->he->ms, &apd->arch); apd->dbg = dso__debuginfo(map__dso(apd->he->ms.map)); } @@ -1813,7 +1836,7 @@ int annotation_br_cntr_abbr_list(char **str, struct evsel *evsel, bool header) struct evsel *pos; struct strbuf sb; - if (evsel->evlist->nr_br_cntr <= 0) + if (evlist__nr_br_cntr(evsel->evlist) <= 0) return -ENOTSUP; strbuf_init(&sb, /*hint=*/ 0); diff --git a/tools/perf/util/annotate.h b/tools/perf/util/annotate.h index 1aa6df7d16187e..fa08d09b80f760 100644 --- a/tools/perf/util/annotate.h +++ b/tools/perf/util/annotate.h @@ -584,5 +584,6 @@ int annotation_br_cntr_entry(char **str, int br_cntr_nr, u64 *br_cntr, int num_aggr, struct evsel *evsel); int annotation_br_cntr_abbr_list(char **str, struct evsel *evsel, bool header); -int thread__get_arch(struct thread *thread, const struct arch **parch); + +int map_symbol__get_arch(struct map_symbol *ms, const struct arch **parch); #endif /* __PERF_ANNOTATE_H */ diff --git a/tools/perf/util/arm-spe.c b/tools/perf/util/arm-spe.c index 552f063f126e67..401aab529309cb 100644 --- a/tools/perf/util/arm-spe.c +++ b/tools/perf/util/arm-spe.c @@ -1605,6 +1605,10 @@ static u64 **arm_spe__alloc_metadata(struct perf_record_auxtrace_info *info, hdr_sz = ptr[ARM_SPE_HEADER_SIZE]; *nr_cpu = ptr[ARM_SPE_CPUS_NUM]; + /* nr_cpu is used as a divisor below */ + if (*nr_cpu <= 0) + return NULL; + metadata = calloc(*nr_cpu, sizeof(*metadata)); if (!metadata) return NULL; diff --git a/tools/perf/util/auxtrace.c b/tools/perf/util/auxtrace.c index 4cd2caf5401522..aa749e1c30365a 100644 --- a/tools/perf/util/auxtrace.c +++ b/tools/perf/util/auxtrace.c @@ -191,7 +191,7 @@ void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp, struct evlist *evlist, struct evsel *evsel, int idx) { - bool per_cpu = !perf_cpu_map__has_any_cpu(evlist->core.user_requested_cpus); + bool per_cpu = !perf_cpu_map__has_any_cpu(evlist__core(evlist)->user_requested_cpus); mp->mmap_needed = evsel->needs_auxtrace_mmap; @@ -201,11 +201,11 @@ void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp, mp->idx = idx; if (per_cpu) { - mp->cpu = perf_cpu_map__cpu(evlist->core.all_cpus, idx); - mp->tid = perf_thread_map__pid(evlist->core.threads, 0); + mp->cpu = perf_cpu_map__cpu(evlist__core(evlist)->all_cpus, idx); + mp->tid = perf_thread_map__pid(evlist__core(evlist)->threads, 0); } else { mp->cpu.cpu = -1; - mp->tid = perf_thread_map__pid(evlist->core.threads, idx); + mp->tid = perf_thread_map__pid(evlist__core(evlist)->threads, idx); } } @@ -251,8 +251,12 @@ static int auxtrace_queues__grow(struct auxtrace_queues *queues, { unsigned int nr_queues = queues->nr_queues; struct auxtrace_queue *queue_array; + struct auxtrace_queue *old_array = queues->queue_array; unsigned int i; + if (!new_nr_queues) + return -EINVAL; + if (!nr_queues) nr_queues = AUXTRACE_INIT_NR_QUEUES; @@ -267,16 +271,17 @@ static int auxtrace_queues__grow(struct auxtrace_queues *queues, return -ENOMEM; for (i = 0; i < queues->nr_queues; i++) { - list_splice_tail(&queues->queue_array[i].head, + list_splice_tail(&old_array[i].head, &queue_array[i].head); - queue_array[i].tid = queues->queue_array[i].tid; - queue_array[i].cpu = queues->queue_array[i].cpu; - queue_array[i].set = queues->queue_array[i].set; - queue_array[i].priv = queues->queue_array[i].priv; + queue_array[i].tid = old_array[i].tid; + queue_array[i].cpu = old_array[i].cpu; + queue_array[i].set = old_array[i].set; + queue_array[i].priv = old_array[i].priv; } queues->nr_queues = nr_queues; queues->queue_array = queue_array; + free(old_array); return 0; } @@ -668,10 +673,10 @@ int auxtrace_parse_snapshot_options(struct auxtrace_record *itr, static int evlist__enable_event_idx(struct evlist *evlist, struct evsel *evsel, int idx) { - bool per_cpu_mmaps = !perf_cpu_map__has_any_cpu(evlist->core.user_requested_cpus); + bool per_cpu_mmaps = !perf_cpu_map__has_any_cpu(evlist__core(evlist)->user_requested_cpus); if (per_cpu_mmaps) { - struct perf_cpu evlist_cpu = perf_cpu_map__cpu(evlist->core.all_cpus, idx); + struct perf_cpu evlist_cpu = perf_cpu_map__cpu(evlist__core(evlist)->all_cpus, idx); int cpu_map_idx = perf_cpu_map__idx(evsel->core.cpus, evlist_cpu); if (cpu_map_idx == -1) @@ -1838,7 +1843,7 @@ void perf_session__auxtrace_error_inc(struct perf_session *session, struct perf_record_auxtrace_error *e = &event->auxtrace_error; if (e->type < PERF_AUXTRACE_ERROR_MAX) - session->evlist->stats.nr_auxtrace_errors[e->type] += 1; + evlist__stats(session->evlist)->nr_auxtrace_errors[e->type] += 1; } void events_stats__auxtrace_error_warn(const struct events_stats *stats) diff --git a/tools/perf/util/block-info.c b/tools/perf/util/block-info.c index 8d3a9a661f2676..1135e54f4c7fcc 100644 --- a/tools/perf/util/block-info.c +++ b/tools/perf/util/block-info.c @@ -472,7 +472,7 @@ struct block_report *block_info__create_report(struct evlist *evlist, int *nr_reps) { struct block_report *block_reports; - int nr_hists = evlist->core.nr_entries, i = 0; + int nr_hists = evlist__nr_entries(evlist), i = 0; struct evsel *pos; block_reports = calloc(nr_hists, sizeof(struct block_report)); @@ -483,7 +483,7 @@ struct block_report *block_info__create_report(struct evlist *evlist, struct hists *hists = evsel__hists(pos); process_block_report(hists, &block_reports[i], total_cycles, - block_hpps, nr_hpps, evlist->nr_br_cntr); + block_hpps, nr_hpps, evlist__nr_br_cntr(evlist)); i++; } diff --git a/tools/perf/util/bpf-event.c b/tools/perf/util/bpf-event.c index fa3ebc8ea7f09c..e67f28a8e92bde 100644 --- a/tools/perf/util/bpf-event.c +++ b/tools/perf/util/bpf-event.c @@ -969,7 +969,8 @@ void __bpf_event__print_bpf_prog_info(struct perf_bpil *info_linear, { struct bpf_prog_info *info = &info_linear->info; __u64 required_arrays = (1UL << PERF_BPIL_JITED_KSYMS) | - (1UL << PERF_BPIL_JITED_FUNC_LENS); + (1UL << PERF_BPIL_JITED_FUNC_LENS) | + (1UL << PERF_BPIL_PROG_TAGS); __u32 *prog_lens; __u64 *prog_addrs; char name[KSYM_NAME_LEN]; diff --git a/tools/perf/util/bpf-filter.c b/tools/perf/util/bpf-filter.c index 1a2e7b388d57d3..bcd81084e3420a 100644 --- a/tools/perf/util/bpf-filter.c +++ b/tools/perf/util/bpf-filter.c @@ -629,24 +629,20 @@ struct perf_bpf_filter_expr *perf_bpf_filter_expr__new(enum perf_bpf_filter_term static bool check_bpf_filter_capable(void) { - bool used_root; + int fd; - if (perf_cap__capable(CAP_BPF, &used_root)) + if (perf_cap__capable(CAP_BPF)) return true; - if (!used_root) { - /* Check if root already pinned the filter programs and maps */ - int fd = get_pinned_fd("filters"); - - if (fd >= 0) { - close(fd); - return true; - } + /* Check if root already pinned the filter programs and maps */ + fd = get_pinned_fd("filters"); + if (fd >= 0) { + close(fd); + return true; } - pr_err("Error: BPF filter only works for %s!\n" - "\tPlease run 'perf record --setup-filter pin' as root first.\n", - used_root ? "root" : "users with the CAP_BPF capability"); + pr_err("Error: BPF filter only works for users with the CAP_BPF capability!\n" + "\tPlease run 'perf record --setup-filter pin' as root first.\n"); return false; } diff --git a/tools/perf/util/bpf_counter.c b/tools/perf/util/bpf_counter.c index 34b6b0da18b738..9362e45e17ceb9 100644 --- a/tools/perf/util/bpf_counter.c +++ b/tools/perf/util/bpf_counter.c @@ -443,7 +443,7 @@ static int bperf_check_target(struct evsel *evsel, } else if (target->tid) { *filter_type = BPERF_FILTER_PID; *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads); - } else if (target->pid || evsel->evlist->workload.pid != -1) { + } else if (target->pid || evlist__workload_pid(evsel->evlist) != -1) { *filter_type = BPERF_FILTER_TGID; *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads); } else { diff --git a/tools/perf/util/bpf_counter_cgroup.c b/tools/perf/util/bpf_counter_cgroup.c index e1ce5aa3b9578e..4e5f4b9dd44284 100644 --- a/tools/perf/util/bpf_counter_cgroup.c +++ b/tools/perf/util/bpf_counter_cgroup.c @@ -104,7 +104,7 @@ static int bperf_load_program(struct evlist *evlist) set_max_rlimit(); - if (nr_cgroups == 0 || evlist->core.nr_entries % nr_cgroups != 0) { + if (nr_cgroups == 0 || evlist__nr_entries(evlist) % nr_cgroups != 0) { pr_err("Invalid cgroup or event count\n"); return -EINVAL; } @@ -116,7 +116,7 @@ static int bperf_load_program(struct evlist *evlist) pr_err("Failed to open cgroup skeleton\n"); return -1; } - setup_rodata(skel, evlist->core.nr_entries); + setup_rodata(skel, evlist__nr_entries(evlist)); err = bperf_cgroup_bpf__load(skel); if (err) { @@ -127,12 +127,12 @@ static int bperf_load_program(struct evlist *evlist) err = -1; cgrp_switch = evsel__new(&cgrp_switch_attr); - if (evsel__open_per_cpu(cgrp_switch, evlist->core.all_cpus, -1) < 0) { + if (evsel__open_per_cpu(cgrp_switch, evlist__core(evlist)->all_cpus, -1) < 0) { pr_err("Failed to open cgroup switches event\n"); goto out; } - perf_cpu_map__for_each_cpu(cpu, i, evlist->core.all_cpus) { + perf_cpu_map__for_each_cpu(cpu, i, evlist__core(evlist)->all_cpus) { link = bpf_program__attach_perf_event(skel->progs.on_cgrp_switch, FD(cgrp_switch, i)); if (IS_ERR(link)) { @@ -197,7 +197,7 @@ static int bperf_load_program(struct evlist *evlist) */ { struct evsel *leader; - int num_events = evlist->core.nr_entries / nr_cgroups; + int num_events = evlist__nr_entries(evlist) / nr_cgroups; evlist__for_each_entry(evlist, evsel) { leader = evlist__find_evsel(evlist, evsel->core.idx % num_events); @@ -258,7 +258,7 @@ static int bperf_cgrp__sync_counters(struct evlist *evlist) unsigned int idx; int prog_fd = bpf_program__fd(skel->progs.trigger_read); - perf_cpu_map__for_each_cpu(cpu, idx, evlist->core.all_cpus) + perf_cpu_map__for_each_cpu(cpu, idx, evlist__core(evlist)->all_cpus) bperf_trigger_reading(prog_fd, cpu.cpu); return 0; @@ -336,7 +336,7 @@ static int bperf_cgrp__destroy(struct evsel *evsel) return 0; bperf_cgroup_bpf__destroy(skel); - evsel__delete(cgrp_switch); // it'll destroy on_switch progs too + evsel__put(cgrp_switch); // it'll destroy on_switch progs too return 0; } diff --git a/tools/perf/util/bpf_ftrace.c b/tools/perf/util/bpf_ftrace.c index c456d24efa308c..abeafd406e8e07 100644 --- a/tools/perf/util/bpf_ftrace.c +++ b/tools/perf/util/bpf_ftrace.c @@ -59,13 +59,13 @@ int perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace) /* don't need to set cpu filter for system-wide mode */ if (ftrace->target.cpu_list) { - ncpus = perf_cpu_map__nr(ftrace->evlist->core.user_requested_cpus); + ncpus = perf_cpu_map__nr(evlist__core(ftrace->evlist)->user_requested_cpus); bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus); skel->rodata->has_cpu = 1; } if (target__has_task(&ftrace->target) || target__none(&ftrace->target)) { - ntasks = perf_thread_map__nr(ftrace->evlist->core.threads); + ntasks = perf_thread_map__nr(evlist__core(ftrace->evlist)->threads); bpf_map__set_max_entries(skel->maps.task_filter, ntasks); skel->rodata->has_task = 1; } @@ -87,7 +87,8 @@ int perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace) fd = bpf_map__fd(skel->maps.cpu_filter); for (i = 0; i < ncpus; i++) { - cpu = perf_cpu_map__cpu(ftrace->evlist->core.user_requested_cpus, i).cpu; + cpu = perf_cpu_map__cpu( + evlist__core(ftrace->evlist)->user_requested_cpus, i).cpu; bpf_map_update_elem(fd, &cpu, &val, BPF_ANY); } } @@ -99,7 +100,7 @@ int perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace) fd = bpf_map__fd(skel->maps.task_filter); for (i = 0; i < ntasks; i++) { - pid = perf_thread_map__pid(ftrace->evlist->core.threads, i); + pid = perf_thread_map__pid(evlist__core(ftrace->evlist)->threads, i); bpf_map_update_elem(fd, &pid, &val, BPF_ANY); } } diff --git a/tools/perf/util/bpf_lock_contention.c b/tools/perf/util/bpf_lock_contention.c index b1cfa63a488fa6..c20bd075664e18 100644 --- a/tools/perf/util/bpf_lock_contention.c +++ b/tools/perf/util/bpf_lock_contention.c @@ -223,11 +223,11 @@ int lock_contention_prepare(struct lock_contention *con) if (target__has_cpu(target)) { skel->rodata->has_cpu = 1; - ncpus = perf_cpu_map__nr(evlist->core.user_requested_cpus); + ncpus = perf_cpu_map__nr(evlist__core(evlist)->user_requested_cpus); } if (target__has_task(target)) { skel->rodata->has_task = 1; - ntasks = perf_thread_map__nr(evlist->core.threads); + ntasks = perf_thread_map__nr(evlist__core(evlist)->threads); } if (con->filters->nr_types) { skel->rodata->has_type = 1; @@ -334,7 +334,7 @@ int lock_contention_prepare(struct lock_contention *con) fd = bpf_map__fd(skel->maps.cpu_filter); for (i = 0; i < ncpus; i++) { - cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, i).cpu; + cpu = perf_cpu_map__cpu(evlist__core(evlist)->user_requested_cpus, i).cpu; bpf_map_update_elem(fd, &cpu, &val, BPF_ANY); } } @@ -346,13 +346,13 @@ int lock_contention_prepare(struct lock_contention *con) fd = bpf_map__fd(skel->maps.task_filter); for (i = 0; i < ntasks; i++) { - pid = perf_thread_map__pid(evlist->core.threads, i); + pid = perf_thread_map__pid(evlist__core(evlist)->threads, i); bpf_map_update_elem(fd, &pid, &val, BPF_ANY); } } - if (target__none(target) && evlist->workload.pid > 0) { - u32 pid = evlist->workload.pid; + if (target__none(target) && evlist__workload_pid(evlist) > 0) { + u32 pid = evlist__workload_pid(evlist); u8 val = 1; fd = bpf_map__fd(skel->maps.task_filter); diff --git a/tools/perf/util/bpf_off_cpu.c b/tools/perf/util/bpf_off_cpu.c index a3b699a5322f1e..c4639f6a577665 100644 --- a/tools/perf/util/bpf_off_cpu.c +++ b/tools/perf/util/bpf_off_cpu.c @@ -1,23 +1,25 @@ // SPDX-License-Identifier: GPL-2.0 -#include "util/bpf_counter.h" -#include "util/debug.h" -#include "util/evsel.h" -#include "util/evlist.h" -#include "util/off_cpu.h" -#include "util/perf-hooks.h" -#include "util/record.h" -#include "util/session.h" -#include "util/target.h" -#include "util/cpumap.h" -#include "util/thread_map.h" -#include "util/cgroup.h" -#include "util/strlist.h" +#include + #include #include #include -#include +#include "bpf_counter.h" #include "bpf_skel/off_cpu.skel.h" +#include "cgroup.h" +#include "cpumap.h" +#include "debug.h" +#include "evlist.h" +#include "evsel.h" +#include "off_cpu.h" +#include "parse-events.h" +#include "perf-hooks.h" +#include "record.h" +#include "session.h" +#include "strlist.h" +#include "target.h" +#include "thread_map.h" #define MAX_STACKS 32 #define MAX_PROC 4096 @@ -71,13 +73,13 @@ static void off_cpu_start(void *arg) /* update task filter for the given workload */ if (skel->rodata->has_task && skel->rodata->uses_tgid && - perf_thread_map__pid(evlist->core.threads, 0) != -1) { + perf_thread_map__pid(evlist__core(evlist)->threads, 0) != -1) { int fd; u32 pid; u8 val = 1; fd = bpf_map__fd(skel->maps.task_filter); - pid = perf_thread_map__pid(evlist->core.threads, 0); + pid = perf_thread_map__pid(evlist__core(evlist)->threads, 0); bpf_map_update_elem(fd, &pid, &val, BPF_ANY); } @@ -166,7 +168,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target, /* don't need to set cpu filter for system-wide mode */ if (target->cpu_list) { - ncpus = perf_cpu_map__nr(evlist->core.user_requested_cpus); + ncpus = perf_cpu_map__nr(evlist__core(evlist)->user_requested_cpus); bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus); skel->rodata->has_cpu = 1; } @@ -197,7 +199,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target, skel->rodata->has_task = 1; skel->rodata->uses_tgid = 1; } else if (target__has_task(target)) { - ntasks = perf_thread_map__nr(evlist->core.threads); + ntasks = perf_thread_map__nr(evlist__core(evlist)->threads); bpf_map__set_max_entries(skel->maps.task_filter, ntasks); skel->rodata->has_task = 1; } else if (target__none(target)) { @@ -207,7 +209,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target, } if (evlist__first(evlist)->cgrp) { - ncgrps = evlist->core.nr_entries - 1; /* excluding a dummy */ + ncgrps = evlist__nr_entries(evlist) - 1; /* excluding a dummy */ bpf_map__set_max_entries(skel->maps.cgroup_filter, ncgrps); if (!cgroup_is_v2("perf_event")) @@ -238,7 +240,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target, fd = bpf_map__fd(skel->maps.cpu_filter); for (i = 0; i < ncpus; i++) { - cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, i).cpu; + cpu = perf_cpu_map__cpu(evlist__core(evlist)->user_requested_cpus, i).cpu; bpf_map_update_elem(fd, &cpu, &val, BPF_ANY); } } @@ -267,7 +269,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target, fd = bpf_map__fd(skel->maps.task_filter); for (i = 0; i < ntasks; i++) { - pid = perf_thread_map__pid(evlist->core.threads, i); + pid = perf_thread_map__pid(evlist__core(evlist)->threads, i); bpf_map_update_elem(fd, &pid, &val, BPF_ANY); } } diff --git a/tools/perf/util/bpf_trace_augment.c b/tools/perf/util/bpf_trace_augment.c index 9e706f0fa53d4a..a9cf2a77ded17e 100644 --- a/tools/perf/util/bpf_trace_augment.c +++ b/tools/perf/util/bpf_trace_augment.c @@ -1,11 +1,11 @@ #include #include -#include "util/debug.h" -#include "util/evlist.h" -#include "util/trace_augment.h" - #include "bpf_skel/augmented_raw_syscalls.skel.h" +#include "debug.h" +#include "evlist.h" +#include "parse-events.h" +#include "trace_augment.h" static struct augmented_raw_syscalls_bpf *skel; static struct evsel *bpf_output; diff --git a/tools/perf/util/cap.c b/tools/perf/util/cap.c index ac6d1d9a523d9f..272bd8255ff12c 100644 --- a/tools/perf/util/cap.c +++ b/tools/perf/util/cap.c @@ -12,7 +12,7 @@ #define MAX_LINUX_CAPABILITY_U32S _LINUX_CAPABILITY_U32S_3 -bool perf_cap__capable(int cap, bool *used_root) +bool perf_cap__capable(int cap) { struct __user_cap_header_struct header = { .version = _LINUX_CAPABILITY_VERSION_3, @@ -21,7 +21,6 @@ bool perf_cap__capable(int cap, bool *used_root) struct __user_cap_data_struct data[MAX_LINUX_CAPABILITY_U32S] = {}; __u32 cap_val; - *used_root = false; while (syscall(SYS_capget, &header, &data[0]) == -1) { /* Retry, first attempt has set the header.version correctly. */ if (errno == EINVAL && header.version != _LINUX_CAPABILITY_VERSION_3 && @@ -29,7 +28,6 @@ bool perf_cap__capable(int cap, bool *used_root) continue; pr_debug2("capget syscall failed (%m) fall back on root check\n"); - *used_root = true; return geteuid() == 0; } diff --git a/tools/perf/util/cap.h b/tools/perf/util/cap.h index c1b8ac033ccc58..bf09fb20c77934 100644 --- a/tools/perf/util/cap.h +++ b/tools/perf/util/cap.h @@ -18,7 +18,6 @@ #define CAP_BPF 39 #endif -/* Query if a capability is supported, used_root is set if the fallback root check was used. */ -bool perf_cap__capable(int cap, bool *used_root); +bool perf_cap__capable(int cap); #endif /* __PERF_CAP_H */ diff --git a/tools/perf/util/capstone.c b/tools/perf/util/capstone.c index 5ad537fea4360e..74213daf878623 100644 --- a/tools/perf/util/capstone.c +++ b/tools/perf/util/capstone.c @@ -24,6 +24,13 @@ #include "symbol.h" #include "thread.h" +#if CS_VERSION_MAJOR < 5 +#define CS_ARCH_RISCV 15 +#define CS_MODE_RISCV32 1 +#define CS_MODE_RISCV64 2 +#define CS_MODE_RISCVC 4 +#endif + #ifdef LIBCAPSTONE_DLOPEN static void *perf_cs_dll_handle(void) { @@ -302,6 +309,7 @@ static void print_capstone_detail(struct cs_insn *insn, char *buf, size_t len, for (i = 0; i < insn->detail->x86.op_count; i++) { struct cs_x86_op *op = &insn->detail->x86.operands[i]; u64 orig_addr; + struct map *found_map = NULL; if (op->type != X86_OP_MEM) continue; @@ -317,19 +325,22 @@ static void print_capstone_detail(struct cs_insn *insn, char *buf, size_t len, if (dso__kernel(map__dso(map))) { /* * The kernel maps can be split into sections, let's - * find the map first and the search the symbol. + * find the map first and then search the symbol. */ - map = maps__find(map__kmaps(map), addr); - if (map == NULL) + found_map = maps__find(map__kmaps(map), addr); + if (found_map == NULL) continue; + map = found_map; } /* convert it to map-relative address for search */ addr = map__map_ip(map, addr); sym = map__find_symbol(map, addr); - if (sym == NULL) + if (sym == NULL) { + map__put(found_map); continue; + } if (addr == sym->start) { scnprintf(buf, len, "\t# %"PRIx64" <%s>", @@ -338,6 +349,7 @@ static void print_capstone_detail(struct cs_insn *insn, char *buf, size_t len, scnprintf(buf, len, "\t# %"PRIx64" <%s+%#"PRIx64">", orig_addr, sym->name, addr - sym->start); } + map__put(found_map); break; } } @@ -380,7 +392,7 @@ int symbol__disassemble_capstone(const char *filename, struct symbol *sym, char disasm_buf[512]; struct disasm_line *dl; bool disassembler_style = false; - uint16_t e_machine; + uint16_t e_machine = EM_NONE; bool is_big_endian = false; if (args->options->objdump_path) @@ -411,9 +423,22 @@ int symbol__disassemble_capstone(const char *filename, struct symbol *sym, !strcmp(args->options->disassembler_style, "att")) disassembler_style = true; - e_machine = thread__e_machine_endian(args->ms->thread, - /*machine=*/NULL, - /*e_flags=*/NULL, &is_big_endian); + if (args->ms->thread) { + e_machine = thread__e_machine_endian(args->ms->thread, + /*machine=*/NULL, + /*e_flags=*/NULL, &is_big_endian); + } else if (dso) { + struct maps *kmaps = (map && dso__kernel(dso)) ? map__kmaps(map) : NULL; + struct machine *kmap_machine = kmaps ? maps__machine(kmaps) : NULL; + + e_machine = dso__e_machine_endian(dso, kmap_machine, /*e_flags=*/NULL, + &is_big_endian); + } + if (!e_machine || e_machine == EM_NONE) { + e_machine = thread__e_machine_endian(NULL, + /*machine=*/NULL, + /*e_flags=*/NULL, &is_big_endian); + } if (capstone_init(e_machine, &handle, is_64bit, is_big_endian, disassembler_style) < 0) goto err; @@ -506,7 +531,7 @@ int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, struct disasm_line *dl; u32 *line; bool disassembler_style = false; - uint16_t e_machine; + uint16_t e_machine = EM_NONE; bool is_big_endian = false; if (args->options->objdump_path) @@ -526,9 +551,22 @@ int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, !strcmp(args->options->disassembler_style, "att")) disassembler_style = true; - e_machine = thread__e_machine_endian(args->ms->thread, - /*machine=*/NULL, - /*e_flags=*/NULL, &is_big_endian); + if (args->ms->thread) { + e_machine = thread__e_machine_endian(args->ms->thread, + /*machine=*/NULL, + /*e_flags=*/NULL, &is_big_endian); + } else if (dso) { + struct maps *kmaps = (map && dso__kernel(dso)) ? map__kmaps(map) : NULL; + struct machine *kmap_machine = kmaps ? maps__machine(kmaps) : NULL; + + e_machine = dso__e_machine_endian(dso, kmap_machine, /*e_flags=*/NULL, + &is_big_endian); + } + if (!e_machine || e_machine == EM_NONE) { + e_machine = thread__e_machine_endian(NULL, + /*machine=*/NULL, + /*e_flags=*/NULL, &is_big_endian); + } if (capstone_init(e_machine, &handle, is_64bit, is_big_endian, disassembler_style) < 0) goto err; diff --git a/tools/perf/util/cgroup.c b/tools/perf/util/cgroup.c index 1b5664d1481f53..c7be16a7915e63 100644 --- a/tools/perf/util/cgroup.c +++ b/tools/perf/util/cgroup.c @@ -367,7 +367,7 @@ int parse_cgroups(const struct option *opt, const char *str, char *s; int ret, i; - if (list_empty(&evlist->core.entries)) { + if (list_empty(&evlist__core(evlist)->entries)) { fprintf(stderr, "must define events before cgroups\n"); return -1; } @@ -423,7 +423,7 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro int ret = -1; int prefix_len; - if (evlist->core.nr_entries == 0) { + if (evlist__nr_entries(evlist) == 0) { fprintf(stderr, "must define events before cgroups\n"); return -EINVAL; } @@ -436,11 +436,11 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro } /* save original events and init evlist */ - evlist__splice_list_tail(orig_list, &evlist->core.entries); - evlist->core.nr_entries = 0; + evlist__splice_list_tail(orig_list, &evlist__core(evlist)->entries); + evlist__core(evlist)->nr_entries = 0; - orig_metric_events = evlist->metric_events; - metricgroup__rblist_init(&evlist->metric_events); + orig_metric_events = *evlist__metric_events(evlist); + metricgroup__rblist_init(evlist__metric_events(evlist)); if (has_pattern_string(str)) prefix_len = match_cgroups(str); @@ -469,7 +469,7 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro /* copy the list and set to the new cgroup. */ evlist__for_each_entry(orig_list, pos) { - struct evsel *evsel = evsel__clone(/*dest=*/NULL, pos); + struct evsel *evsel = evsel__clone(pos); if (evsel == NULL) goto out_err; @@ -503,15 +503,15 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro nr_cgroups++; if (metricgroup__copy_metric_events(tmp_list, cgrp, - &evlist->metric_events, + evlist__metric_events(evlist), &orig_metric_events) < 0) goto out_err; - evlist__splice_list_tail(evlist, &tmp_list->core.entries); - tmp_list->core.nr_entries = 0; + evlist__splice_list_tail(evlist, &evlist__core(tmp_list)->entries); + evlist__core(tmp_list)->nr_entries = 0; } - if (list_empty(&evlist->core.entries)) { + if (list_empty(&evlist__core(evlist)->entries)) { fprintf(stderr, "no cgroup matched: %s\n", str); goto out_err; } @@ -520,8 +520,8 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro cgrp_event_expanded = true; out_err: - evlist__delete(orig_list); - evlist__delete(tmp_list); + evlist__put(orig_list); + evlist__put(tmp_list); metricgroup__rblist_exit(&orig_metric_events); release_cgroup_list(); diff --git a/tools/perf/util/compress.h b/tools/perf/util/compress.h index 6cfecfca16f24d..ec6c38129e2482 100644 --- a/tools/perf/util/compress.h +++ b/tools/perf/util/compress.h @@ -54,7 +54,8 @@ int zstd_fini(struct zstd_data *data); ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_t dst_size, void *src, size_t src_size, size_t max_record_size, - size_t process_header(void *record, size_t increment)); + ssize_t process_header(void *record, size_t dst_size, + size_t data_size)); size_t zstd_decompress_stream(struct zstd_data *data, void *src, size_t src_size, void *dst, size_t dst_size); @@ -75,7 +76,8 @@ ssize_t zstd_compress_stream_to_records(struct zstd_data *data __maybe_unused, void *dst __maybe_unused, size_t dst_size __maybe_unused, void *src __maybe_unused, size_t src_size __maybe_unused, size_t max_record_size __maybe_unused, - size_t process_header(void *record, size_t increment) __maybe_unused) + ssize_t process_header(void *record, size_t dst_size, + size_t data_size) __maybe_unused) { return 0; } diff --git a/tools/perf/util/cs-etm.c b/tools/perf/util/cs-etm.c index 5d0664ff73b791..fc3d4ebd1fbae8 100644 --- a/tools/perf/util/cs-etm.c +++ b/tools/perf/util/cs-etm.c @@ -71,6 +71,7 @@ struct cs_etm_auxtrace { int num_cpu; u64 latest_kernel_timestamp; u32 auxtrace_type; + u32 branches_filter; u64 branches_sample_type; u64 branches_id; u64 instructions_sample_type; @@ -645,6 +646,8 @@ static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq, queue->tid); tidq->decode_thread = machine__findnew_thread(&etm->session->machines.host, -1, queue->tid); + if (!tidq->frontend_thread || !tidq->decode_thread) + goto out; tidq->packet = zalloc(sizeof(struct cs_etm_packet)); if (!tidq->packet) @@ -679,6 +682,8 @@ out_free: zfree(&tidq->prev_packet); zfree(&tidq->packet); out: + thread__zput(tidq->frontend_thread); + thread__zput(tidq->decode_thread); return rc; } @@ -1523,8 +1528,7 @@ cs_etm__get_trace(struct cs_etm_queue *etmq) etmq->buf_used = 0; etmq->buf_len = aux_buffer->size; etmq->buf = aux_buffer->data; - - return etmq->buf_len; + return 0; } /* @@ -1691,8 +1695,9 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, { int ret = 0; struct cs_etm_auxtrace *etm = etmq->etm; - struct perf_sample sample = {.ip = 0,}; + struct perf_sample sample; union perf_event *event = tidq->event_buf; + struct dummy_branch_stack { u64 nr; u64 hw_idx; @@ -1700,6 +1705,11 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, } dummy_bs; u64 ip; + if (etm->branches_filter && + !(etm->branches_filter & tidq->prev_packet->flags)) + return 0; + + perf_sample__init(&sample, /*all=*/true); ip = cs_etm__last_executed_instr(tidq->prev_packet); event->sample.header.type = PERF_RECORD_SAMPLE; @@ -1752,6 +1762,7 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, "CS ETM Trace: failed to deliver instruction event, error %d\n", ret); + perf_sample__exit(&sample); return ret; } @@ -2077,6 +2088,45 @@ static int cs_etm__end_block(struct cs_etm_queue *etmq, return 0; } + +static int cs_etm__flush_stack_cb(struct thread *thread, + void *data __maybe_unused) +{ + thread_stack__flush(thread); + return 0; +} + +static void cs_etm__flush_machine_stack(struct cs_etm_queue *etmq, pid_t pid) +{ + struct machine *machine; + + machine = machines__find(&etmq->etm->session->machines, pid); + if (machine) + machine__for_each_thread(machine, cs_etm__flush_stack_cb, NULL); +} + +static void cs_etm__flush_all_stack(struct cs_etm_queue *etmq) +{ + enum cs_etm_pid_fmt pid_fmt = cs_etm__get_pid_fmt(etmq); + + if (!etmq->etm->synth_opts.last_branch) + return; + + switch (pid_fmt) { + case CS_ETM_PIDFMT_CTXTID2: + /* Clear the guest stack if virtualization is supported */ + cs_etm__flush_machine_stack(etmq, DEFAULT_GUEST_KERNEL_ID); + fallthrough; + case CS_ETM_PIDFMT_CTXTID: + cs_etm__flush_machine_stack(etmq, HOST_KERNEL_ID); + break; + case CS_ETM_PIDFMT_NONE: + default: + break; + + } +} + /* * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue * if need be. @@ -2088,20 +2138,33 @@ static int cs_etm__get_data_block(struct cs_etm_queue *etmq) { int ret; - if (!etmq->buf_len) { - ret = cs_etm__get_trace(etmq); - if (ret <= 0) - return ret; - /* - * We cannot assume consecutive blocks in the data file - * are contiguous, reset the decoder to force re-sync. - */ - ret = cs_etm_decoder__reset(etmq->decoder); - if (ret) - return ret; - } + /* The current block is not finished */ + if (etmq->buf_len) + return 1; + + ret = cs_etm__get_trace(etmq); + if (ret < 0) + return ret; - return etmq->buf_len; + /* No more buffer to read */ + if (!etmq->buf_len) + return 0; + + /* + * We cannot assume consecutive blocks in the data file + * are contiguous, reset the decoder to force re-sync. + */ + ret = cs_etm_decoder__reset(etmq->decoder); + if (ret) + return ret; + + /* + * Since the decoder is reset, this causes a global trace + * discontinuity. Flush all thread stacks. + */ + cs_etm__flush_all_stack(etmq); + + return 1; } static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, @@ -3557,6 +3620,16 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event, etm->synth_opts.callchain = false; } + if (etm->synth_opts.calls) + etm->branches_filter |= PERF_IP_FLAG_CALL | + PERF_IP_FLAG_TRACE_BEGIN | + PERF_IP_FLAG_TRACE_END; + + if (etm->synth_opts.returns) + etm->branches_filter |= PERF_IP_FLAG_RETURN | + PERF_IP_FLAG_TRACE_BEGIN | + PERF_IP_FLAG_TRACE_END; + etm->session = session; etm->num_cpu = num_cpu; diff --git a/tools/perf/util/data-convert-bt.c b/tools/perf/util/data-convert-bt.c index 5ff46bfcd0e19a..e222371394c01a 100644 --- a/tools/perf/util/data-convert-bt.c +++ b/tools/perf/util/data-convert-bt.c @@ -1362,7 +1362,7 @@ static void cleanup_events(struct perf_session *session) zfree(&evsel->priv); } - evlist__delete(evlist); + evlist__put(evlist); session->evlist = NULL; } diff --git a/tools/perf/util/data-convert-json.c b/tools/perf/util/data-convert-json.c index 40412c3dbdb25c..40888b7c446710 100644 --- a/tools/perf/util/data-convert-json.c +++ b/tools/perf/util/data-convert-json.c @@ -258,6 +258,7 @@ static int process_sample_event(const struct perf_tool *tool, trace_seq_init(&s); tep_print_field(&s, sample->raw_data, fields[i]); output_json_key_string(out, true, 3, fields[i]->name, s.buffer); + trace_seq_destroy(&s); i++; } diff --git a/tools/perf/util/dso.c b/tools/perf/util/dso.c index 2309196d8df311..42bfe30a3b518e 100644 --- a/tools/perf/util/dso.c +++ b/tools/perf/util/dso.c @@ -395,7 +395,9 @@ int dso__decompress_kmodule_path(struct dso *dso, const char *name, { int fd = decompress_kmodule(dso, name, pathname, len); - close(fd); + /* decompress_kmodule() returns -1 on failure, don't close(-1) */ + if (fd >= 0) + close(fd); return fd >= 0 ? 0 : -1; } @@ -582,9 +584,18 @@ static char *dso__get_filename(struct dso *dso, const char *root_dir, goto out; if (!is_regular_file(name)) { + struct stat st; char *new_name; - if (errno != ENOENT || dso__nsinfo(dso) == NULL) + /* + * errno only reflects the failure reason when stat() itself + * failed: a successful stat() on a non-regular file (e.g. a + * directory) leaves a stale errno, which a previous failed + * iteration of the try_to_open_dso() fallback loop may have + * set to ENOENT. + */ + if (stat(name, &st) == 0 || errno != ENOENT || + dso__nsinfo(dso) == NULL) goto out; new_name = dso__filename_with_chroot(dso, name); @@ -640,10 +651,13 @@ static int __open_dso(struct dso *dso, struct machine *machine) mutex_lock(dso__lock(dso)); name = dso__get_filename(dso, machine ? machine->root_dir : "", &decomp); - if (name) + if (name) { fd = do_open(name); - else + } else { + if (errno == 0) + errno = ENOENT; fd = -errno; + } if (decomp) unlink(name); @@ -1000,7 +1014,20 @@ static ssize_t dso_cache__memcpy(struct dso_cache *cache, u64 offset, u8 *data, u64 size, bool out) { u64 cache_offset = offset - cache->offset; - u64 cache_size = min(cache->size - cache_offset, size); + u64 cache_size; + + /* + * The RB tree matches using DSO__DATA_CACHE_SIZE, but a short + * pread may leave cache->size smaller. For a regular file a + * short pread only happens at end-of-file, so an offset past + * the valid data is EOF: return 0, matching what a direct + * pread() at that offset would return, and cached_io() then + * stops its read loop. + */ + if (cache_offset >= cache->size) + return 0; + + cache_size = min(cache->size - cache_offset, size); if (out) memcpy(data, cache->data + cache_offset, cache_size); @@ -1024,7 +1051,7 @@ static ssize_t file_read(struct dso *dso, struct machine *machine, if (dso__data(dso)->fd < 0) { dso__data(dso)->status = DSO_DATA_STATUS_ERROR; - ret = -errno; + ret = dso__data(dso)->fd; goto out; } @@ -1146,8 +1173,8 @@ static int file_size(struct dso *dso, struct machine *machine) try_to_open_dso(dso, machine); if (dso__data(dso)->fd < 0) { - ret = -errno; dso__data(dso)->status = DSO_DATA_STATUS_ERROR; + ret = dso__data(dso)->fd; goto out; } @@ -2011,7 +2038,12 @@ const u8 *dso__read_symbol(struct dso *dso, const char *symfs_filename, errno = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; return NULL; } - assert(len <= info_linear->info.jited_prog_len); + if (len > info_linear->info.jited_prog_len) { + pr_debug("BPF symbol length %zu exceeds jited_prog_len %u\n", + len, info_linear->info.jited_prog_len); + errno = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; + return NULL; + } *out_buf_len = len; return (const u8 *)(uintptr_t)(info_linear->info.jited_prog_insns); #else diff --git a/tools/perf/util/dso.h b/tools/perf/util/dso.h index 2916b954a804b6..55c4aaa53c382e 100644 --- a/tools/perf/util/dso.h +++ b/tools/perf/util/dso.h @@ -9,6 +9,7 @@ #include #include #include +#include #include "build-id.h" #include "debuginfo.h" #include "mutex.h" @@ -20,6 +21,40 @@ struct perf_env; #define DSO__NAME_KALLSYMS "[kernel.kallsyms]" #define DSO__NAME_KCORE "[kernel.kcore]" +#define DSO__NAME_GUEST_KALLSYMS "[guest.kernel.kallsyms]" +#define DSO__NAME_GUEST_KALLSYMS_PID_PREFIX "[guest.kernel.kallsyms." + +/* + * Validate names of the form "[guest.kernel.kallsyms.]", where + * is the PID of the guest VM and varies per guest, so it + * cannot be matched with strcmp() against a fixed string. + * + * Every character after the fixed prefix must be a decimal digit, + * with ']' immediately terminating the digit run and nothing + * following it. This rules out '/', "..", or any other character + * being smuggled into the name. + */ +static inline bool is_guest_kallsyms_pid_name(const char *name) +{ + const size_t prefix_len = sizeof(DSO__NAME_GUEST_KALLSYMS_PID_PREFIX) - 1; + size_t digits; + + if (strncmp(name, DSO__NAME_GUEST_KALLSYMS_PID_PREFIX, prefix_len) != 0) + return false; + + digits = strspn(name + prefix_len, "0123456789"); + if (digits == 0) + return false; + + /* ']' must terminate the digit run, with nothing trailing it */ + if (name[prefix_len + digits] != ']') + return false; + + if (name[prefix_len + digits + 1] != '\0') + return false; + + return true; +} /** * enum dso_binary_type - The kind of DSO generally associated with a memory @@ -924,8 +959,28 @@ static inline bool dso__is_kcore(const struct dso *dso) static inline bool dso__is_kallsyms(const struct dso *dso) { enum dso_binary_type bt = dso__binary_type(dso); + const char *name; + + if (bt == DSO_BINARY_TYPE__KALLSYMS || bt == DSO_BINARY_TYPE__GUEST_KALLSYMS) + return true; + + if (bt != DSO_BINARY_TYPE__NOT_FOUND) + return false; + + if (!dso__kernel(dso)) + return false; + + name = dso__long_name(dso); + if (!name) + return false; + + if (!strcmp(name, DSO__NAME_KALLSYMS)) + return true; + + if (!strcmp(name, DSO__NAME_GUEST_KALLSYMS)) + return true; - return bt == DSO_BINARY_TYPE__KALLSYMS || bt == DSO_BINARY_TYPE__GUEST_KALLSYMS; + return is_guest_kallsyms_pid_name(name); } bool dso__is_object_file(const struct dso *dso); diff --git a/tools/perf/util/evlist.c b/tools/perf/util/evlist.c index 1a238b245b3a00..eb7c0d7be064e3 100644 --- a/tools/perf/util/evlist.c +++ b/tools/perf/util/evlist.c @@ -5,67 +5,69 @@ * Parts came from builtin-{top,stat,record}.c, see those files for further * copyright notes. */ -#include +#include "evlist.h" + #include #include -#include -#include "cpumap.h" -#include "util/mmap.h" -#include "thread_map.h" -#include "target.h" -#include "dwarf-regs.h" -#include "evlist.h" -#include "evsel.h" -#include "record.h" -#include "debug.h" -#include "units.h" -#include "bpf_counter.h" -#include // page_size -#include "affinity.h" -#include "../perf.h" -#include "asm/bug.h" -#include "bpf-event.h" -#include "util/event.h" -#include "util/string2.h" -#include "util/perf_api_probe.h" -#include "util/evsel_fprintf.h" -#include "util/pmu.h" -#include "util/sample.h" -#include "util/bpf-filter.h" -#include "util/stat.h" -#include "util/util.h" -#include "util/env.h" -#include "util/intel-tpebs.h" -#include "util/metricgroup.h" -#include "util/strbuf.h" #include -#include -#include #include -#include "parse-events.h" -#include - #include -#include -#include -#include -#include -#include - #include +#include #include #include -#include #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include // page_size +#include +#include +#include #include #include -#include #include +#include -#include +#include "../perf.h" +#include "affinity.h" +#include "asm/bug.h" +#include "bpf-event.h" +#include "bpf-filter.h" +#include "bpf_counter.h" +#include "cpumap.h" +#include "debug.h" +#include "dwarf-regs.h" +#include "env.h" +#include "event.h" +#include "evsel.h" +#include "evsel_fprintf.h" +#include "intel-tpebs.h" +#include "metricgroup.h" +#include "mmap.h" +#include "parse-events.h" +#include "perf_api_probe.h" +#include "pmu.h" +#include "pmus.h" +#include "record.h" +#include "sample.h" +#include "stat.h" +#include "strbuf.h" +#include "string2.h" +#include "target.h" +#include "thread_map.h" +#include "units.h" +#include "util.h" #ifdef LACKS_SIGQUEUE_PROTOTYPE int sigqueue(pid_t pid, int sig, const union sigval value); @@ -74,29 +76,31 @@ int sigqueue(pid_t pid, int sig, const union sigval value); #define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y)) #define SID(e, x, y) xyarray__entry(e->core.sample_id, x, y) -void evlist__init(struct evlist *evlist, struct perf_cpu_map *cpus, - struct perf_thread_map *threads) -{ - perf_evlist__init(&evlist->core); - perf_evlist__set_maps(&evlist->core, cpus, threads); - evlist->workload.pid = -1; - evlist->bkw_mmap_state = BKW_MMAP_NOTREADY; - evlist->ctl_fd.fd = -1; - evlist->ctl_fd.ack = -1; - evlist->ctl_fd.pos = -1; - evlist->nr_br_cntr = -1; - metricgroup__rblist_init(&evlist->metric_events); - INIT_LIST_HEAD(&evlist->deferred_samples); -} +static void event_enable_timer__exit(struct event_enable_timer **ep); struct evlist *evlist__new(void) { - struct evlist *evlist = zalloc(sizeof(*evlist)); - - if (evlist != NULL) - evlist__init(evlist, NULL, NULL); - - return evlist; + struct evlist *result; + RC_STRUCT(evlist) *evlist; + + evlist = zalloc(sizeof(*evlist)); + if (ADD_RC_CHK(result, evlist)) { + perf_evlist__init(evlist__core(result)); + perf_evlist__set_maps(evlist__core(result), /*cpus=*/NULL, /*threads=*/NULL); + evlist__set_workload_pid(result, -1); + evlist__set_bkw_mmap_state(result, BKW_MMAP_NOTREADY); + evlist__set_ctl_fd_fd(result, -1); + evlist__set_ctl_fd_ack(result, -1); + evlist__set_ctl_fd_pos(result, -1); + evlist__set_nr_br_cntr(result, -1); + metricgroup__rblist_init(evlist__metric_events(result)); + INIT_LIST_HEAD(&evlist->deferred_samples); + refcount_set(evlist__refcnt(result), 1); + } else { + free(evlist); + result = NULL; + } + return result; } struct evlist *evlist__new_default(const struct target *target, bool sample_callchains) @@ -104,7 +108,6 @@ struct evlist *evlist__new_default(const struct target *target, bool sample_call struct evlist *evlist = evlist__new(); bool can_profile_kernel; struct perf_pmu *pmu = NULL; - struct evsel *evsel; char buf[256]; int err; @@ -131,14 +134,16 @@ struct evlist *evlist__new_default(const struct target *target, bool sample_call } /* If there is only 1 event a sample identifier isn't necessary. */ - if (evlist->core.nr_entries > 1) { + if (evlist__nr_entries(evlist) > 1) { + struct evsel *evsel; + evlist__for_each_entry(evlist, evsel) evsel__set_sample_id(evsel, /*can_sample_identifier=*/false); } return evlist; out_err: - evlist__delete(evlist); + evlist__put(evlist); return NULL; } @@ -147,13 +152,23 @@ struct evlist *evlist__new_dummy(void) struct evlist *evlist = evlist__new(); if (evlist && evlist__add_dummy(evlist)) { - evlist__delete(evlist); + evlist__put(evlist); evlist = NULL; } return evlist; } +struct evlist *evlist__get(struct evlist *evlist) +{ + struct evlist *result; + + if (RC_CHK_GET(result, evlist)) + refcount_inc(evlist__refcnt(evlist)); + + return result; +} + /** * evlist__set_id_pos - set the positions of event ids. * @evlist: selected event list @@ -165,8 +180,8 @@ void evlist__set_id_pos(struct evlist *evlist) { struct evsel *first = evlist__first(evlist); - evlist->id_pos = first->id_pos; - evlist->is_pos = first->is_pos; + RC_CHK_ACCESS(evlist)->id_pos = first->id_pos; + RC_CHK_ACCESS(evlist)->is_pos = first->is_pos; } static void evlist__update_id_pos(struct evlist *evlist) @@ -185,49 +200,85 @@ static void evlist__purge(struct evlist *evlist) evlist__for_each_entry_safe(evlist, n, pos) { list_del_init(&pos->core.node); + if (pos->evlist) { + if (!RC_CHK_EQUAL(pos->evlist, evlist)) { + evlist__put(pos->evlist); + } else { + refcount_dec_and_test(evlist__refcnt(pos->evlist)); + RC_CHK_PUT(pos->evlist); + } + } pos->evlist = NULL; - evsel__delete(pos); + evsel__put(pos); } - evlist->core.nr_entries = 0; + evlist__core(evlist)->nr_entries = 0; } -void evlist__exit(struct evlist *evlist) +static void evlist__exit(struct evlist *evlist) { - metricgroup__rblist_exit(&evlist->metric_events); - event_enable_timer__exit(&evlist->eet); - zfree(&evlist->mmap); - zfree(&evlist->overwrite_mmap); - perf_evlist__exit(&evlist->core); + metricgroup__rblist_exit(evlist__metric_events(evlist)); + event_enable_timer__exit(&RC_CHK_ACCESS(evlist)->eet); + free(evlist__mmap(evlist)); + free(evlist__overwrite_mmap(evlist)); + perf_evlist__exit(evlist__core(evlist)); } -void evlist__delete(struct evlist *evlist) +void evlist__put(struct evlist *evlist) { + struct evsel *evsel; + unsigned int count, ref_cnt; + if (evlist == NULL) return; + if (refcount_dec_and_test(evlist__refcnt(evlist))) + goto out_delete; + +retry: + count = refcount_read(evlist__refcnt(evlist)); + ref_cnt = count; + evlist__for_each_entry(evlist, evsel) { + if (RC_CHK_EQUAL(evsel->evlist, evlist) && count && + refcount_read(&evsel->refcnt) == 1) + count--; + } + if (refcount_read(evlist__refcnt(evlist)) != ref_cnt) + goto retry; + + if (count != 0) { + /* + * Not the last reference except for back references from + * evsels. + */ + RC_CHK_PUT(evlist); + return; + } +out_delete: evlist__free_stats(evlist); - evlist__munmap(evlist); + evlist__do_munmap(evlist); evlist__close(evlist); evlist__purge(evlist); evlist__exit(evlist); - free(evlist); + RC_CHK_FREE(evlist); } void evlist__add(struct evlist *evlist, struct evsel *entry) { - perf_evlist__add(&evlist->core, &entry->core); - entry->evlist = evlist; + perf_evlist__add(evlist__core(evlist), &entry->core); + evlist__put(entry->evlist); + entry->evlist = evlist__get(evlist); entry->tracking = !entry->core.idx; - if (evlist->core.nr_entries == 1) + if (evlist__nr_entries(evlist) == 1) evlist__set_id_pos(evlist); } void evlist__remove(struct evlist *evlist, struct evsel *evsel) { + perf_evlist__remove(evlist__core(evlist), &evsel->core); + evlist__put(evsel->evlist); evsel->evlist = NULL; - perf_evlist__remove(&evlist->core, &evsel->core); } void evlist__splice_list_tail(struct evlist *evlist, struct list_head *list) @@ -276,7 +327,7 @@ out: static void evlist__set_leader(struct evlist *evlist) { - perf_evlist__set_leader(&evlist->core); + perf_evlist__set_leader(evlist__core(evlist)); } static struct evsel *evlist__dummy_event(struct evlist *evlist) @@ -290,7 +341,7 @@ static struct evsel *evlist__dummy_event(struct evlist *evlist) .sample_period = 1, }; - return evsel__new_idx(&attr, evlist->core.nr_entries); + return evsel__new_idx(&attr, evlist__nr_entries(evlist)); } int evlist__add_dummy(struct evlist *evlist) @@ -379,8 +430,8 @@ static bool evlist__use_affinity(struct evlist *evlist) struct perf_cpu_map *used_cpus = NULL; bool ret = false; - if (evlist->no_affinity || !evlist->core.user_requested_cpus || - cpu_map__is_dummy(evlist->core.user_requested_cpus)) + if (evlist__no_affinity(evlist) || !evlist__core(evlist)->user_requested_cpus || + cpu_map__is_dummy(evlist__core(evlist)->user_requested_cpus)) return false; evlist__for_each_entry(evlist, pos) { @@ -435,7 +486,7 @@ void evlist_cpu_iterator__init(struct evlist_cpu_iterator *itr, struct evlist *e .evsel = NULL, .cpu_map_idx = 0, .evlist_cpu_map_idx = 0, - .evlist_cpu_map_nr = perf_cpu_map__nr(evlist->core.all_cpus), + .evlist_cpu_map_nr = perf_cpu_map__nr(evlist__core(evlist)->all_cpus), .cpu = (struct perf_cpu){ .cpu = -1}, .affinity = NULL, }; @@ -451,7 +502,7 @@ void evlist_cpu_iterator__init(struct evlist_cpu_iterator *itr, struct evlist *e itr->affinity = &itr->saved_affinity; } itr->evsel = evlist__first(evlist); - itr->cpu = perf_cpu_map__cpu(evlist->core.all_cpus, 0); + itr->cpu = perf_cpu_map__cpu(evlist__core(evlist)->all_cpus, 0); if (itr->affinity) affinity__set(itr->affinity, itr->cpu.cpu); itr->cpu_map_idx = perf_cpu_map__idx(itr->evsel->core.cpus, itr->cpu); @@ -486,7 +537,7 @@ void evlist_cpu_iterator__next(struct evlist_cpu_iterator *evlist_cpu_itr) if (evlist_cpu_itr->evlist_cpu_map_idx < evlist_cpu_itr->evlist_cpu_map_nr) { evlist_cpu_itr->evsel = evlist__first(evlist_cpu_itr->container); evlist_cpu_itr->cpu = - perf_cpu_map__cpu(evlist_cpu_itr->container->core.all_cpus, + perf_cpu_map__cpu(evlist__core(evlist_cpu_itr->container)->all_cpus, evlist_cpu_itr->evlist_cpu_map_idx); if (evlist_cpu_itr->affinity) affinity__set(evlist_cpu_itr->affinity, evlist_cpu_itr->cpu.cpu); @@ -513,7 +564,7 @@ static int evsel__strcmp(struct evsel *pos, char *evsel_name) return !evsel__name_is(pos, evsel_name); } -static int evlist__is_enabled(struct evlist *evlist) +static bool evlist__is_enabled(struct evlist *evlist) { struct evsel *pos; @@ -570,10 +621,7 @@ static void __evlist__disable(struct evlist *evlist, char *evsel_name, bool excl * If we disabled only single event, we need to check * the enabled state of the evlist manually. */ - if (evsel_name) - evlist->enabled = evlist__is_enabled(evlist); - else - evlist->enabled = false; + evlist__set_enabled(evlist, evsel_name ? evlist__is_enabled(evlist) : false); } void evlist__disable(struct evlist *evlist) @@ -624,7 +672,7 @@ static void __evlist__enable(struct evlist *evlist, char *evsel_name, bool excl_ * so the toggle can work properly and toggle to * 'disabled' state. */ - evlist->enabled = true; + evlist__set_enabled(evlist, true); } void evlist__enable(struct evlist *evlist) @@ -644,23 +692,24 @@ void evlist__enable_evsel(struct evlist *evlist, char *evsel_name) void evlist__toggle_enable(struct evlist *evlist) { - (evlist->enabled ? evlist__disable : evlist__enable)(evlist); + (evlist__enabled(evlist) ? evlist__disable : evlist__enable)(evlist); } int evlist__add_pollfd(struct evlist *evlist, int fd) { - return perf_evlist__add_pollfd(&evlist->core, fd, NULL, POLLIN, fdarray_flag__default); + return perf_evlist__add_pollfd(evlist__core(evlist), fd, NULL, POLLIN, + fdarray_flag__default); } int evlist__filter_pollfd(struct evlist *evlist, short revents_and_mask) { - return perf_evlist__filter_pollfd(&evlist->core, revents_and_mask); + return perf_evlist__filter_pollfd(evlist__core(evlist), revents_and_mask); } #ifdef HAVE_EVENTFD_SUPPORT int evlist__add_wakeup_eventfd(struct evlist *evlist, int fd) { - return perf_evlist__add_pollfd(&evlist->core, fd, NULL, POLLIN, + return perf_evlist__add_pollfd(evlist__core(evlist), fd, NULL, POLLIN, fdarray_flag__nonfilterable | fdarray_flag__non_perf_event); } @@ -668,7 +717,7 @@ int evlist__add_wakeup_eventfd(struct evlist *evlist, int fd) int evlist__poll(struct evlist *evlist, int timeout) { - return perf_evlist__poll(&evlist->core, timeout); + return perf_evlist__poll(evlist__core(evlist), timeout); } struct perf_sample_id *evlist__id2sid(struct evlist *evlist, u64 id) @@ -678,7 +727,7 @@ struct perf_sample_id *evlist__id2sid(struct evlist *evlist, u64 id) int hash; hash = hash_64(id, PERF_EVLIST__HLIST_BITS); - head = &evlist->core.heads[hash]; + head = &evlist__core(evlist)->heads[hash]; hlist_for_each_entry(sid, head, node) if (sid->id == id) @@ -691,7 +740,7 @@ struct evsel *evlist__id2evsel(struct evlist *evlist, u64 id) { struct perf_sample_id *sid; - if (evlist->core.nr_entries == 1 || !id) + if (evlist__nr_entries(evlist) == 1 || !id) return evlist__first(evlist); sid = evlist__id2sid(evlist, id); @@ -726,13 +775,13 @@ static int evlist__event2id(struct evlist *evlist, union perf_event *event, u64 n = (event->header.size - sizeof(event->header)) >> 3; if (event->header.type == PERF_RECORD_SAMPLE) { - if (evlist->id_pos >= n) + if (evlist__id_pos(evlist) >= n) return -1; - *id = array[evlist->id_pos]; + *id = array[evlist__id_pos(evlist)]; } else { - if (evlist->is_pos > n) + if (evlist__is_pos(evlist) > n) return -1; - n -= evlist->is_pos; + n -= evlist__is_pos(evlist); *id = array[n]; } return 0; @@ -746,7 +795,7 @@ struct evsel *evlist__event2evsel(struct evlist *evlist, union perf_event *event int hash; u64 id; - if (evlist->core.nr_entries == 1) + if (evlist__nr_entries(evlist) == 1) return first; if (!first->core.attr.sample_id_all && @@ -761,7 +810,7 @@ struct evsel *evlist__event2evsel(struct evlist *evlist, union perf_event *event return first; hash = hash_64(id, PERF_EVLIST__HLIST_BITS); - head = &evlist->core.heads[hash]; + head = &evlist__core(evlist)->heads[hash]; hlist_for_each_entry(sid, head, node) { if (sid->id == id) @@ -774,11 +823,11 @@ static int evlist__set_paused(struct evlist *evlist, bool value) { int i; - if (!evlist->overwrite_mmap) + if (!evlist__overwrite_mmap(evlist)) return 0; - for (i = 0; i < evlist->core.nr_mmaps; i++) { - int fd = evlist->overwrite_mmap[i].core.fd; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + int fd = evlist__overwrite_mmap(evlist)[i].core.fd; int err; if (fd < 0) @@ -804,20 +853,20 @@ static void evlist__munmap_nofree(struct evlist *evlist) { int i; - if (evlist->mmap) - for (i = 0; i < evlist->core.nr_mmaps; i++) - perf_mmap__munmap(&evlist->mmap[i].core); + if (evlist__mmap(evlist)) + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) + perf_mmap__munmap(&evlist__mmap(evlist)[i].core); - if (evlist->overwrite_mmap) - for (i = 0; i < evlist->core.nr_mmaps; i++) - perf_mmap__munmap(&evlist->overwrite_mmap[i].core); + if (evlist__overwrite_mmap(evlist)) + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) + perf_mmap__munmap(&evlist__overwrite_mmap(evlist)[i].core); } -void evlist__munmap(struct evlist *evlist) +void evlist__do_munmap(struct evlist *evlist) { evlist__munmap_nofree(evlist); - zfree(&evlist->mmap); - zfree(&evlist->overwrite_mmap); + zfree(&RC_CHK_ACCESS(evlist)->mmap); + zfree(&RC_CHK_ACCESS(evlist)->overwrite_mmap); } static void perf_mmap__unmap_cb(struct perf_mmap *map) @@ -831,12 +880,12 @@ static struct mmap *evlist__alloc_mmap(struct evlist *evlist, bool overwrite) { int i; - struct mmap *map = calloc(evlist->core.nr_mmaps, sizeof(struct mmap)); + struct mmap *map = calloc(evlist__core(evlist)->nr_mmaps, sizeof(struct mmap)); if (!map) return NULL; - for (i = 0; i < evlist->core.nr_mmaps; i++) { + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { struct perf_mmap *prev = i ? &map[i - 1].core : NULL; /* @@ -854,41 +903,73 @@ static struct mmap *evlist__alloc_mmap(struct evlist *evlist, return map; } +static struct evlist *from_list_start(struct perf_evlist *core) +{ +#ifdef REFCNT_CHECKING + RC_STRUCT(evlist) *core_evlist = container_of(core, RC_STRUCT(evlist), core); + struct evlist *evlist; + + if (ADD_RC_CHK(evlist, core_evlist)) + refcount_inc(evlist__refcnt(evlist)); + + return evlist; +#else + return container_of(core, struct evlist, core); +#endif +} + +static void from_list_end(struct evlist *evlist __maybe_unused) +{ +#ifdef REFCNT_CHECKING + evlist__put(evlist); +#endif +} + static void perf_evlist__mmap_cb_idx(struct perf_evlist *_evlist, struct perf_evsel *_evsel, struct perf_mmap_param *_mp, int idx) { - struct evlist *evlist = container_of(_evlist, struct evlist, core); + struct evlist *evlist = from_list_start(_evlist); struct mmap_params *mp = container_of(_mp, struct mmap_params, core); struct evsel *evsel = container_of(_evsel, struct evsel, core); + if (!evlist) + return; + auxtrace_mmap_params__set_idx(&mp->auxtrace_mp, evlist, evsel, idx); + + from_list_end(evlist); } static struct perf_mmap* perf_evlist__mmap_cb_get(struct perf_evlist *_evlist, bool overwrite, int idx) { - struct evlist *evlist = container_of(_evlist, struct evlist, core); + struct evlist *evlist = from_list_start(_evlist); struct mmap *maps; - maps = overwrite ? evlist->overwrite_mmap : evlist->mmap; + if (!evlist) + return NULL; + + maps = overwrite ? evlist__overwrite_mmap(evlist) : evlist__mmap(evlist); if (!maps) { maps = evlist__alloc_mmap(evlist, overwrite); - if (!maps) + if (!maps) { + from_list_end(evlist); return NULL; + } if (overwrite) { - evlist->overwrite_mmap = maps; - if (evlist->bkw_mmap_state == BKW_MMAP_NOTREADY) + RC_CHK_ACCESS(evlist)->overwrite_mmap = maps; + if (evlist__bkw_mmap_state(evlist) == BKW_MMAP_NOTREADY) evlist__toggle_bkw_mmap(evlist, BKW_MMAP_RUNNING); } else { - evlist->mmap = maps; + RC_CHK_ACCESS(evlist)->mmap = maps; } } - + from_list_end(evlist); return &maps[idx].core; } @@ -1045,16 +1126,16 @@ int evlist__mmap_ex(struct evlist *evlist, unsigned int pages, .mmap = perf_evlist__mmap_cb_mmap, }; - evlist->core.mmap_len = evlist__mmap_size(pages); - pr_debug("mmap size %zuB\n", evlist->core.mmap_len); + evlist__core(evlist)->mmap_len = evlist__mmap_size(pages); + pr_debug("mmap size %zuB\n", evlist__core(evlist)->mmap_len); - auxtrace_mmap_params__init(&mp.auxtrace_mp, evlist->core.mmap_len, + auxtrace_mmap_params__init(&mp.auxtrace_mp, evlist__core(evlist)->mmap_len, auxtrace_pages, auxtrace_overwrite); - return perf_evlist__mmap_ops(&evlist->core, &ops, &mp.core); + return perf_evlist__mmap_ops(evlist__core(evlist), &ops, &mp.core); } -int evlist__mmap(struct evlist *evlist, unsigned int pages) +int evlist__do_mmap(struct evlist *evlist, unsigned int pages) { return evlist__mmap_ex(evlist, pages, 0, false, 0, PERF_AFFINITY_SYS, 1, 0); } @@ -1096,9 +1177,9 @@ int evlist__create_maps(struct evlist *evlist, struct target *target) if (!cpus) goto out_delete_threads; - evlist->core.has_user_cpus = !!target->cpu_list; + evlist__core(evlist)->has_user_cpus = !!target->cpu_list; - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); /* as evlist now has references, put count here */ perf_cpu_map__put(cpus); @@ -1238,15 +1319,15 @@ bool evlist__valid_sample_type(struct evlist *evlist) { struct evsel *pos; - if (evlist->core.nr_entries == 1) + if (evlist__nr_entries(evlist) == 1) return true; - if (evlist->id_pos < 0 || evlist->is_pos < 0) + if (evlist__id_pos(evlist) < 0 || evlist__is_pos(evlist) < 0) return false; evlist__for_each_entry(evlist, pos) { - if (pos->id_pos != evlist->id_pos || - pos->is_pos != evlist->is_pos) + if (pos->id_pos != evlist__id_pos(evlist) || + pos->is_pos != evlist__is_pos(evlist)) return false; } @@ -1257,18 +1338,18 @@ u64 __evlist__combined_sample_type(struct evlist *evlist) { struct evsel *evsel; - if (evlist->combined_sample_type) - return evlist->combined_sample_type; + if (RC_CHK_ACCESS(evlist)->combined_sample_type) + return RC_CHK_ACCESS(evlist)->combined_sample_type; evlist__for_each_entry(evlist, evsel) - evlist->combined_sample_type |= evsel->core.attr.sample_type; + RC_CHK_ACCESS(evlist)->combined_sample_type |= evsel->core.attr.sample_type; - return evlist->combined_sample_type; + return RC_CHK_ACCESS(evlist)->combined_sample_type; } u64 evlist__combined_sample_type(struct evlist *evlist) { - evlist->combined_sample_type = 0; + RC_CHK_ACCESS(evlist)->combined_sample_type = 0; return __evlist__combined_sample_type(evlist); } @@ -1345,7 +1426,7 @@ void evlist__update_br_cntr(struct evlist *evlist) evlist__new_abbr_name(evsel->abbr_name); } } - evlist->nr_br_cntr = i; + evlist__set_nr_br_cntr(evlist, i); } bool evlist__valid_read_format(struct evlist *evlist) @@ -1395,11 +1476,6 @@ bool evlist__sample_id_all(struct evlist *evlist) return first->core.attr.sample_id_all; } -void evlist__set_selected(struct evlist *evlist, struct evsel *evsel) -{ - evlist->selected = evsel; -} - void evlist__close(struct evlist *evlist) { struct evsel *evsel; @@ -1416,7 +1492,7 @@ void evlist__close(struct evlist *evlist) perf_evsel__free_fd(&evsel->core); perf_evsel__free_id(&evsel->core); } - perf_evlist__reset_id_hash(&evlist->core); + perf_evlist__reset_id_hash(evlist__core(evlist)); } static int evlist__create_syswide_maps(struct evlist *evlist) @@ -1443,7 +1519,7 @@ static int evlist__create_syswide_maps(struct evlist *evlist) return -ENOMEM; } - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); perf_thread_map__put(threads); perf_cpu_map__put(cpus); return 0; @@ -1458,7 +1534,8 @@ int evlist__open(struct evlist *evlist) * Default: one fd per CPU, all threads, aka systemwide * as sys_perf_event_open(cpu = -1, thread = -1) is EINVAL */ - if (evlist->core.threads == NULL && evlist->core.user_requested_cpus == NULL) { + if (evlist__core(evlist)->threads == NULL && + evlist__core(evlist)->user_requested_cpus == NULL) { err = evlist__create_syswide_maps(evlist); if (err < 0) goto out_err; @@ -1485,7 +1562,7 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const int child_ready_pipe[2], go_pipe[2]; char bf; - evlist->workload.cork_fd = -1; + evlist__set_workload_cork_fd(evlist, -1); if (pipe(child_ready_pipe) < 0) { perror("failed to create 'ready' pipe"); @@ -1497,13 +1574,13 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const goto out_close_ready_pipe; } - evlist->workload.pid = fork(); - if (evlist->workload.pid < 0) { + evlist__set_workload_pid(evlist, fork()); + if (evlist__workload_pid(evlist) < 0) { perror("failed to fork"); goto out_close_pipes; } - if (!evlist->workload.pid) { + if (!evlist__workload_pid(evlist)) { int ret; if (pipe_output) @@ -1569,12 +1646,13 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const } if (target__none(target)) { - if (evlist->core.threads == NULL) { + if (evlist__core(evlist)->threads == NULL) { fprintf(stderr, "FATAL: evlist->threads need to be set at this point (%s:%d).\n", __func__, __LINE__); goto out_close_pipes; } - perf_thread_map__set_pid(evlist->core.threads, 0, evlist->workload.pid); + perf_thread_map__set_pid(evlist__core(evlist)->threads, 0, + evlist__workload_pid(evlist)); } close(child_ready_pipe[1]); @@ -1588,7 +1666,7 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const } fcntl(go_pipe[1], F_SETFD, FD_CLOEXEC); - evlist->workload.cork_fd = go_pipe[1]; + evlist__set_workload_cork_fd(evlist, go_pipe[1]); close(child_ready_pipe[0]); return 0; @@ -1603,18 +1681,18 @@ out_close_ready_pipe: int evlist__start_workload(struct evlist *evlist) { - if (evlist->workload.cork_fd >= 0) { + if (evlist__workload_cork_fd(evlist) >= 0) { char bf = 0; int ret; /* * Remove the cork, let it rip! */ - ret = write(evlist->workload.cork_fd, &bf, 1); + ret = write(evlist__workload_cork_fd(evlist), &bf, 1); if (ret < 0) perror("unable to write to pipe"); - close(evlist->workload.cork_fd); - evlist->workload.cork_fd = -1; + close(evlist__workload_cork_fd(evlist)); + evlist__set_workload_cork_fd(evlist, -1); return ret; } @@ -1625,10 +1703,10 @@ void evlist__cancel_workload(struct evlist *evlist) { int status; - if (evlist->workload.cork_fd >= 0) { - close(evlist->workload.cork_fd); - evlist->workload.cork_fd = -1; - waitpid(evlist->workload.pid, &status, WNOHANG); + if (evlist__workload_cork_fd(evlist) >= 0) { + close(evlist__workload_cork_fd(evlist)); + evlist__set_workload_cork_fd(evlist, -1); + waitpid(evlist__workload_pid(evlist), &status, WNOHANG); } } @@ -1722,7 +1800,8 @@ out_default: int evlist__strerror_mmap(struct evlist *evlist, int err, char *buf, size_t size) { - int pages_attempted = evlist->core.mmap_len / 1024, pages_max_per_user, printed = 0; + int pages_attempted = evlist__core(evlist)->mmap_len / 1024; + int pages_max_per_user, printed = 0; switch (err) { case EPERM: @@ -1765,7 +1844,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel) list_move_tail(&evsel->core.node, &move); } - list_splice(&move, &evlist->core.entries); + list_splice(&move, &evlist__core(evlist)->entries); } struct evsel *evlist__get_tracking_event(struct evlist *evlist) @@ -1807,7 +1886,7 @@ struct evsel *evlist__findnew_tracking_event(struct evlist *evlist, bool system_ evlist__set_tracking_event(evlist, evsel); } else if (system_wide) { - perf_evlist__go_system_wide(&evlist->core, &evsel->core); + perf_evlist__go_system_wide(evlist__core(evlist), &evsel->core); } return evsel; @@ -1829,14 +1908,14 @@ struct evsel *evlist__find_evsel_by_str(struct evlist *evlist, const char *str) void evlist__toggle_bkw_mmap(struct evlist *evlist, enum bkw_mmap_state state) { - enum bkw_mmap_state old_state = evlist->bkw_mmap_state; + enum bkw_mmap_state old_state = evlist__bkw_mmap_state(evlist); enum action { NONE, PAUSE, RESUME, } action = NONE; - if (!evlist->overwrite_mmap) + if (!evlist__overwrite_mmap(evlist)) return; switch (old_state) { @@ -1866,7 +1945,7 @@ void evlist__toggle_bkw_mmap(struct evlist *evlist, enum bkw_mmap_state state) WARN_ONCE(1, "Shouldn't get there\n"); } - evlist->bkw_mmap_state = state; + evlist__set_bkw_mmap_state(evlist, state); switch (action) { case PAUSE: @@ -2044,40 +2123,41 @@ int evlist__initialize_ctlfd(struct evlist *evlist, int fd, int ack) return 0; } - evlist->ctl_fd.pos = perf_evlist__add_pollfd(&evlist->core, fd, NULL, POLLIN, - fdarray_flag__nonfilterable | - fdarray_flag__non_perf_event); - if (evlist->ctl_fd.pos < 0) { - evlist->ctl_fd.pos = -1; + evlist__set_ctl_fd_pos(evlist, + perf_evlist__add_pollfd(evlist__core(evlist), fd, NULL, POLLIN, + fdarray_flag__nonfilterable | + fdarray_flag__non_perf_event)); + if (evlist__ctl_fd_pos(evlist) < 0) { + evlist__set_ctl_fd_pos(evlist, -1); pr_err("Failed to add ctl fd entry: %m\n"); return -1; } - evlist->ctl_fd.fd = fd; - evlist->ctl_fd.ack = ack; + evlist__set_ctl_fd_fd(evlist, fd); + evlist__set_ctl_fd_ack(evlist, ack); return 0; } bool evlist__ctlfd_initialized(struct evlist *evlist) { - return evlist->ctl_fd.pos >= 0; + return evlist__ctl_fd_pos(evlist) >= 0; } int evlist__finalize_ctlfd(struct evlist *evlist) { - struct pollfd *entries = evlist->core.pollfd.entries; + struct pollfd *entries = evlist__core(evlist)->pollfd.entries; if (!evlist__ctlfd_initialized(evlist)) return 0; - entries[evlist->ctl_fd.pos].fd = -1; - entries[evlist->ctl_fd.pos].events = 0; - entries[evlist->ctl_fd.pos].revents = 0; + entries[evlist__ctl_fd_pos(evlist)].fd = -1; + entries[evlist__ctl_fd_pos(evlist)].events = 0; + entries[evlist__ctl_fd_pos(evlist)].revents = 0; - evlist->ctl_fd.pos = -1; - evlist->ctl_fd.ack = -1; - evlist->ctl_fd.fd = -1; + evlist__set_ctl_fd_pos(evlist, -1); + evlist__set_ctl_fd_ack(evlist, -1); + evlist__set_ctl_fd_fd(evlist, -1); return 0; } @@ -2094,7 +2174,7 @@ static int evlist__ctlfd_recv(struct evlist *evlist, enum evlist_ctl_cmd *cmd, data_size--; do { - err = read(evlist->ctl_fd.fd, &c, 1); + err = read(evlist__ctl_fd_fd(evlist), &c, 1); if (err > 0) { if (c == '\n' || c == '\0') break; @@ -2108,7 +2188,8 @@ static int evlist__ctlfd_recv(struct evlist *evlist, enum evlist_ctl_cmd *cmd, if (errno == EAGAIN || errno == EWOULDBLOCK) err = 0; else - pr_err("Failed to read from ctlfd %d: %m\n", evlist->ctl_fd.fd); + pr_err("Failed to read from ctlfd %d: %m\n", + evlist__ctl_fd_fd(evlist)); } break; } while (1); @@ -2146,13 +2227,13 @@ int evlist__ctlfd_ack(struct evlist *evlist) { int err; - if (evlist->ctl_fd.ack == -1) + if (evlist__ctl_fd_ack(evlist) == -1) return 0; - err = write(evlist->ctl_fd.ack, EVLIST_CTL_CMD_ACK_TAG, + err = write(evlist__ctl_fd_ack(evlist), EVLIST_CTL_CMD_ACK_TAG, sizeof(EVLIST_CTL_CMD_ACK_TAG)); if (err == -1) - pr_err("failed to write to ctl_ack_fd %d: %m\n", evlist->ctl_fd.ack); + pr_err("failed to write to ctl_ack_fd %d: %m\n", evlist__ctl_fd_ack(evlist)); return err; } @@ -2253,8 +2334,8 @@ int evlist__ctlfd_process(struct evlist *evlist, enum evlist_ctl_cmd *cmd) { int err = 0; char cmd_data[EVLIST_CTL_CMD_MAX_LEN]; - int ctlfd_pos = evlist->ctl_fd.pos; - struct pollfd *entries = evlist->core.pollfd.entries; + int ctlfd_pos = evlist__ctl_fd_pos(evlist); + struct pollfd *entries = evlist__core(evlist)->pollfd.entries; if (!evlist__ctlfd_initialized(evlist) || !entries[ctlfd_pos].revents) return 0; @@ -2425,14 +2506,15 @@ int evlist__parse_event_enable_time(struct evlist *evlist, struct record_opts *o goto free_eet_times; } - eet->pollfd_pos = perf_evlist__add_pollfd(&evlist->core, eet->timerfd, NULL, POLLIN, flags); + eet->pollfd_pos = perf_evlist__add_pollfd(evlist__core(evlist), eet->timerfd, + NULL, POLLIN, flags); if (eet->pollfd_pos < 0) { err = eet->pollfd_pos; goto close_timerfd; } eet->evlist = evlist; - evlist->eet = eet; + RC_CHK_ACCESS(evlist)->eet = eet; opts->target.initial_delay = eet->times[0].start; return 0; @@ -2482,7 +2564,7 @@ int event_enable_timer__process(struct event_enable_timer *eet) if (!eet) return 0; - entries = eet->evlist->core.pollfd.entries; + entries = evlist__core(eet->evlist)->pollfd.entries; revents = entries[eet->pollfd_pos].revents; entries[eet->pollfd_pos].revents = 0; @@ -2518,7 +2600,7 @@ int event_enable_timer__process(struct event_enable_timer *eet) return 0; } -void event_enable_timer__exit(struct event_enable_timer **ep) +static void event_enable_timer__exit(struct event_enable_timer **ep) { if (!ep || !*ep) return; @@ -2622,7 +2704,7 @@ void evlist__warn_user_requested_cpus(struct evlist *evlist, const char *cpu_lis } /* Should uniquify be disabled for the evlist? */ -static bool evlist__disable_uniquify(const struct evlist *evlist) +static bool evlist__disable_uniquify(struct evlist *evlist) { struct evsel *counter; struct perf_pmu *last_pmu = NULL; diff --git a/tools/perf/util/evlist.h b/tools/perf/util/evlist.h index e507f5f20ef61b..1997843dca0ef0 100644 --- a/tools/perf/util/evlist.h +++ b/tools/perf/util/evlist.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include "affinity.h" #include "events_stats.h" @@ -56,8 +57,9 @@ enum bkw_mmap_state { struct event_enable_timer; -struct evlist { +DECLARE_RC_STRUCT(evlist) { struct perf_evlist core; + refcount_t refcnt; bool enabled; bool no_affinity; int id_pos; @@ -82,7 +84,7 @@ struct evlist { struct { pthread_t th; volatile int done; - } thread; + } sb_thread; struct { int fd; /* control file descriptor */ int ack; /* ack file descriptor for control commands */ @@ -103,13 +105,232 @@ struct evsel_str_handler { void *handler; }; +static inline struct perf_evlist *evlist__core(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->core; +} + +static inline const struct perf_evlist *evlist__const_core(const struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->core; +} + +static inline int evlist__nr_entries(const struct evlist *evlist) +{ + return evlist__const_core(evlist)->nr_entries; +} + +static inline bool evlist__enabled(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->enabled; +} + +static inline void evlist__set_enabled(struct evlist *evlist, bool enabled) +{ + RC_CHK_ACCESS(evlist)->enabled = enabled; +} + +static inline bool evlist__no_affinity(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->no_affinity; +} + +static inline void evlist__set_no_affinity(struct evlist *evlist, bool no_affinity) +{ + RC_CHK_ACCESS(evlist)->no_affinity = no_affinity; +} + +static inline int evlist__sb_thread_done(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->sb_thread.done; +} + +static inline void evlist__set_sb_thread_done(struct evlist *evlist, int done) +{ + RC_CHK_ACCESS(evlist)->sb_thread.done = done; +} + +static inline pthread_t *evlist__sb_thread_th(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->sb_thread.th; +} + +static inline int evlist__id_pos(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->id_pos; +} + +static inline int evlist__is_pos(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->is_pos; +} + +static inline struct event_enable_timer *evlist__event_enable_timer(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->eet; +} + +static inline enum bkw_mmap_state evlist__bkw_mmap_state(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->bkw_mmap_state; +} + +static inline void evlist__set_bkw_mmap_state(struct evlist *evlist, enum bkw_mmap_state state) +{ + RC_CHK_ACCESS(evlist)->bkw_mmap_state = state; +} + +static inline struct mmap *evlist__mmap(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->mmap; +} + +static inline struct mmap *evlist__overwrite_mmap(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->overwrite_mmap; +} + +static inline struct events_stats *evlist__stats(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->stats; +} + +static inline u64 evlist__first_sample_time(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->first_sample_time; +} + +static inline void evlist__set_first_sample_time(struct evlist *evlist, u64 first) +{ + RC_CHK_ACCESS(evlist)->first_sample_time = first; +} + +static inline u64 evlist__last_sample_time(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->last_sample_time; +} + +static inline void evlist__set_last_sample_time(struct evlist *evlist, u64 last) +{ + RC_CHK_ACCESS(evlist)->last_sample_time = last; +} + +static inline int evlist__nr_br_cntr(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->nr_br_cntr; +} + +static inline void evlist__set_nr_br_cntr(struct evlist *evlist, int nr) +{ + RC_CHK_ACCESS(evlist)->nr_br_cntr = nr; +} + +static inline struct perf_session *evlist__session(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->session; +} + +static inline void evlist__set_session(struct evlist *evlist, struct perf_session *session) +{ + RC_CHK_ACCESS(evlist)->session = session; +} + +static inline void (*evlist__trace_event_sample_raw(struct evlist *evlist)) + (struct evlist *evlist, + union perf_event *event, + struct perf_sample *sample) +{ + return RC_CHK_ACCESS(evlist)->trace_event_sample_raw; +} + +static inline void evlist__set_trace_event_sample_raw(struct evlist *evlist, + void (*fun)(struct evlist *evlist, + union perf_event *event, + struct perf_sample *sample)) +{ + RC_CHK_ACCESS(evlist)->trace_event_sample_raw = fun; +} + +static inline pid_t evlist__workload_pid(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->workload.pid; +} + +static inline void evlist__set_workload_pid(struct evlist *evlist, pid_t pid) +{ + RC_CHK_ACCESS(evlist)->workload.pid = pid; +} + +static inline int evlist__workload_cork_fd(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->workload.cork_fd; +} + +static inline void evlist__set_workload_cork_fd(struct evlist *evlist, int cork_fd) +{ + RC_CHK_ACCESS(evlist)->workload.cork_fd = cork_fd; +} + +static inline int evlist__ctl_fd_fd(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->ctl_fd.fd; +} + +static inline void evlist__set_ctl_fd_fd(struct evlist *evlist, int fd) +{ + RC_CHK_ACCESS(evlist)->ctl_fd.fd = fd; +} + +static inline int evlist__ctl_fd_ack(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->ctl_fd.ack; +} + +static inline void evlist__set_ctl_fd_ack(struct evlist *evlist, int ack) +{ + RC_CHK_ACCESS(evlist)->ctl_fd.ack = ack; +} + +static inline int evlist__ctl_fd_pos(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->ctl_fd.pos; +} + +static inline void evlist__set_ctl_fd_pos(struct evlist *evlist, int pos) +{ + RC_CHK_ACCESS(evlist)->ctl_fd.pos = pos; +} + +static inline refcount_t *evlist__refcnt(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->refcnt; +} + +static inline struct rblist *evlist__metric_events(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->metric_events; +} + +static inline struct list_head *evlist__deferred_samples(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->deferred_samples; +} + +static inline struct evsel *evlist__selected(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->selected; +} + +static inline void evlist__set_selected(struct evlist *evlist, struct evsel *evsel) +{ + RC_CHK_ACCESS(evlist)->selected = evsel; +} + struct evlist *evlist__new(void); struct evlist *evlist__new_default(const struct target *target, bool sample_callchains); struct evlist *evlist__new_dummy(void); -void evlist__init(struct evlist *evlist, struct perf_cpu_map *cpus, - struct perf_thread_map *threads); -void evlist__exit(struct evlist *evlist); -void evlist__delete(struct evlist *evlist); +struct evlist *evlist__get(struct evlist *evlist); +void evlist__put(struct evlist *evlist); void evlist__add(struct evlist *evlist, struct evsel *entry); void evlist__remove(struct evlist *evlist, struct evsel *evsel); @@ -198,8 +419,8 @@ int evlist__mmap_ex(struct evlist *evlist, unsigned int pages, unsigned int auxtrace_pages, bool auxtrace_overwrite, int nr_cblocks, int affinity, int flush, int comp_level); -int evlist__mmap(struct evlist *evlist, unsigned int pages); -void evlist__munmap(struct evlist *evlist); +int evlist__do_mmap(struct evlist *evlist, unsigned int pages); +void evlist__do_munmap(struct evlist *evlist); size_t evlist__mmap_size(unsigned long pages); @@ -211,8 +432,6 @@ void evlist__enable_evsel(struct evlist *evlist, char *evsel_name); void evlist__disable_non_dummy(struct evlist *evlist); void evlist__enable_non_dummy(struct evlist *evlist); -void evlist__set_selected(struct evlist *evlist, struct evsel *evsel); - int evlist__create_maps(struct evlist *evlist, struct target *target); int evlist__apply_filters(struct evlist *evlist, struct evsel **err_evsel, struct target *target); @@ -235,26 +454,26 @@ void evlist__splice_list_tail(struct evlist *evlist, struct list_head *list); static inline bool evlist__empty(struct evlist *evlist) { - return list_empty(&evlist->core.entries); + return list_empty(&evlist__core(evlist)->entries); } static inline struct evsel *evlist__first(struct evlist *evlist) { - struct perf_evsel *evsel = perf_evlist__first(&evlist->core); + struct perf_evsel *evsel = perf_evlist__first(evlist__core(evlist)); return container_of(evsel, struct evsel, core); } static inline struct evsel *evlist__last(struct evlist *evlist) { - struct perf_evsel *evsel = perf_evlist__last(&evlist->core); + struct perf_evsel *evsel = perf_evlist__last(evlist__core(evlist)); return container_of(evsel, struct evsel, core); } static inline int evlist__nr_groups(struct evlist *evlist) { - return perf_evlist__nr_groups(&evlist->core); + return perf_evlist__nr_groups(evlist__core(evlist)); } int evlist__strerror_open(struct evlist *evlist, int err, char *buf, size_t size); @@ -277,7 +496,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel); * @evsel: struct evsel iterator */ #define evlist__for_each_entry(evlist, evsel) \ - __evlist__for_each_entry(&(evlist)->core.entries, evsel) + __evlist__for_each_entry(&evlist__core(evlist)->entries, evsel) /** * __evlist__for_each_entry_continue - continue iteration thru all the evsels @@ -293,7 +512,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel); * @evsel: struct evsel iterator */ #define evlist__for_each_entry_continue(evlist, evsel) \ - __evlist__for_each_entry_continue(&(evlist)->core.entries, evsel) + __evlist__for_each_entry_continue(&evlist__core(evlist)->entries, evsel) /** * __evlist__for_each_entry_from - continue iteration from @evsel (included) @@ -309,7 +528,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel); * @evsel: struct evsel iterator */ #define evlist__for_each_entry_from(evlist, evsel) \ - __evlist__for_each_entry_from(&(evlist)->core.entries, evsel) + __evlist__for_each_entry_from(&evlist__core(evlist)->entries, evsel) /** * __evlist__for_each_entry_reverse - iterate thru all the evsels in reverse order @@ -325,7 +544,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel); * @evsel: struct evsel iterator */ #define evlist__for_each_entry_reverse(evlist, evsel) \ - __evlist__for_each_entry_reverse(&(evlist)->core.entries, evsel) + __evlist__for_each_entry_reverse(&evlist__core(evlist)->entries, evsel) /** * __evlist__for_each_entry_safe - safely iterate thru all the evsels @@ -343,7 +562,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel); * @tmp: struct evsel temp iterator */ #define evlist__for_each_entry_safe(evlist, tmp, evsel) \ - __evlist__for_each_entry_safe(&(evlist)->core.entries, tmp, evsel) + __evlist__for_each_entry_safe(&evlist__core(evlist)->entries, tmp, evsel) /** Iterator state for evlist__for_each_cpu */ struct evlist_cpu_iterator { @@ -449,7 +668,6 @@ int evlist__ctlfd_ack(struct evlist *evlist); int evlist__parse_event_enable_time(struct evlist *evlist, struct record_opts *opts, const char *str, int unset); int event_enable_timer__start(struct event_enable_timer *eet); -void event_enable_timer__exit(struct event_enable_timer **ep); int event_enable_timer__process(struct event_enable_timer *eet); struct evsel *evlist__find_evsel(struct evlist *evlist, int idx); diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c index ea9fa04429f08a..968cd74a9cde9d 100644 --- a/tools/perf/util/evsel.c +++ b/tools/perf/util/evsel.c @@ -16,9 +16,11 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -440,10 +442,11 @@ bool evsel__is_function_event(struct evsel *evsel) #undef FUNCTION_EVENT } -void evsel__init(struct evsel *evsel, +static void evsel__init(struct evsel *evsel, struct perf_event_attr *attr, int idx) { perf_evsel__init(&evsel->core, attr, idx); + refcount_set(&evsel->refcnt, 1); evsel->tracking = !idx; evsel->unit = strdup(""); evsel->scale = 1.0; @@ -525,7 +528,7 @@ static int evsel__copy_config_terms(struct evsel *dst, struct evsel *src) * The assumption is that @orig is not configured nor opened yet. * So we only care about the attributes that can be set while it's parsed. */ -struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig) +struct evsel *evsel__clone(struct evsel *orig) { struct evsel *evsel; @@ -538,11 +541,7 @@ struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig) if (orig->bpf_obj) return NULL; - if (dest) - evsel = dest; - else - evsel = evsel__new(&orig->core.attr); - + evsel = evsel__new(&orig->core.attr); if (evsel == NULL) return NULL; @@ -615,7 +614,13 @@ struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig) evsel->sample_read = orig->sample_read; evsel->collect_stat = orig->collect_stat; evsel->weak_group = orig->weak_group; + evsel->bpf_counter = orig->bpf_counter; evsel->use_config_name = orig->use_config_name; + evsel->skippable = orig->skippable; + evsel->dont_regroup = orig->dont_regroup; + evsel->default_metricgroup = orig->default_metricgroup; + evsel->default_show_events = orig->default_show_events; + evsel->pmu = orig->pmu; evsel->first_wildcard_match = orig->first_wildcard_match; @@ -624,10 +629,14 @@ struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig) evsel->alternate_hw_config = orig->alternate_hw_config; + evsel->retire_lat = orig->retire_lat; + if (evsel->retire_lat) + evsel->retirement_latency = orig->retirement_latency; + return evsel; out_err: - evsel__delete(evsel); + evsel__put(evsel); return NULL; } @@ -686,6 +695,12 @@ out_err: return ERR_PTR(err); } +struct evsel *evsel__get(struct evsel *evsel) +{ + refcount_inc(&evsel->refcnt); + return evsel; +} + #ifdef HAVE_LIBTRACEEVENT struct tep_event *evsel__tp_format(struct evsel *evsel) { @@ -703,7 +718,7 @@ struct tep_event *evsel__tp_format(struct evsel *evsel) tp_format = trace_event__tp_format(evsel->tp_sys, evsel->tp_name); if (IS_ERR(tp_format)) { - int err = -PTR_ERR(evsel->tp_format); + int err = -PTR_ERR(tp_format); errno = err; pr_err("Error getting tracepoint format '%s': %m\n", @@ -2024,7 +2039,7 @@ void evsel__set_priv_destructor(void (*destructor)(void *priv)) evsel__priv_destructor = destructor; } -void evsel__exit(struct evsel *evsel) +static void evsel__exit(struct evsel *evsel) { assert(list_empty(&evsel->core.node)); assert(evsel->evlist == NULL); @@ -2061,11 +2076,14 @@ void evsel__exit(struct evsel *evsel) } } -void evsel__delete(struct evsel *evsel) +void evsel__put(struct evsel *evsel) { if (!evsel) return; + if (!refcount_dec_and_test(&evsel->refcnt)) + return; + evsel__exit(evsel); free(evsel); } @@ -3350,7 +3368,7 @@ static inline bool evsel__has_branch_counters(const struct evsel *evsel) if (!leader || !evsel->evlist) return false; - if (evsel->evlist->nr_br_cntr < 0) + if (evlist__nr_br_cntr(evsel->evlist) < 0) evlist__update_br_cntr(evsel->evlist); if (leader->br_cntr_nr > 0) @@ -3933,6 +3951,95 @@ void *perf_sample__rawptr(struct perf_sample *sample, const char *name) return sample->raw_data + offset; } +void *format_field__get_raw_data(struct tep_format_field *field, struct + perf_sample *sample, bool needs_swap, + u16 *len_out) +{ + int offset = field->offset; + int size = field->size; + + if (field->flags & TEP_FIELD_IS_DYNAMIC) { + unsigned int dynamic_data; + + if (out_of_bounds(field, field->offset, field->size, sample->raw_size)) + return NULL; + + dynamic_data = format_field__intval(field, sample, needs_swap); + + offset = dynamic_data & 0xffff; + size = (dynamic_data >> 16) & 0xffff; + + if (tep_field_is_relative(field->flags)) + offset += field->offset + field->size; + } + + if (out_of_bounds(field, offset, size, sample->raw_size)) + return NULL; + + *len_out = size; + return sample->raw_data + offset; +} + +unsigned long *format_field__get_cpumask(struct tep_format_field *field, + struct perf_sample *sample, + bool needs_swap, u16 *len_out) +{ + u16 len; + void *ptr = format_field__get_raw_data(field, sample, needs_swap, &len); + unsigned long *mask; + struct perf_env *env; + bool target_is_64; + int target_word_size; + int nr_words; + int bit_idx; + int nbits; + + if (!ptr) + return NULL; + + nbits = len * 8; + mask = bitmap_zalloc(nbits ?: 1); + if (!mask) + return NULL; + + env = evsel__env(sample->evsel); + target_is_64 = env ? perf_env__kernel_is_64_bit(env) : (sizeof(void *) == 8); + target_word_size = target_is_64 ? 8 : 4; + nr_words = len / target_word_size; + + for (bit_idx = 0; bit_idx < nbits; bit_idx++) { + int w_idx = bit_idx / (target_word_size * 8); + int bit_in_word = bit_idx % (target_word_size * 8); + bool set = false; + + if (w_idx >= nr_words) + break; + + if (target_is_64) { + u64 word; + memcpy(&word, (unsigned char *)ptr + w_idx * 8, 8); + if (needs_swap) + word = bswap_64(word); + set = (word & (1ULL << bit_in_word)) != 0; + } else { + u32 word32; + memcpy(&word32, (unsigned char *)ptr + w_idx * 4, 4); + if (needs_swap) + word32 = bswap_32(word32); + set = (word32 & (1U << bit_in_word)) != 0; + } + + if (set) { + int host_w_idx = bit_idx / BITS_PER_LONG; + int host_bit_in_word = bit_idx % BITS_PER_LONG; + mask[host_w_idx] |= (1UL << host_bit_in_word); + } + } + + *len_out = len; + return mask; +} + u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample, bool needs_swap) { @@ -4382,7 +4489,7 @@ int evsel__open_strerror(struct evsel *evsel, struct target *target, struct perf_session *evsel__session(struct evsel *evsel) { - return evsel && evsel->evlist ? evsel->evlist->session : NULL; + return evsel && evsel->evlist ? evlist__session(evsel->evlist) : NULL; } struct perf_env *evsel__env(struct evsel *evsel) @@ -4407,7 +4514,7 @@ static int store_evsel_ids(struct evsel *evsel, struct evlist *evlist) thread++) { int fd = FD(evsel, cpu_map_idx, thread); - if (perf_evlist__id_add_fd(&evlist->core, &evsel->core, + if (perf_evlist__id_add_fd(evlist__core(evlist), &evsel->core, cpu_map_idx, thread, fd) < 0) return -1; } diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h index 163fc2b6a7eacc..ba567e3b65c93d 100644 --- a/tools/perf/util/evsel.h +++ b/tools/perf/util/evsel.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -45,6 +46,7 @@ typedef int (evsel__sb_cb_t)(union perf_event *event, void *data); struct evsel { struct perf_evsel core; struct evlist *evlist; + refcount_t refcnt; off_t id_offset; int id_pos; int is_pos; @@ -271,7 +273,7 @@ static inline struct evsel *evsel__new(struct perf_event_attr *attr) return evsel__new_idx(attr, 0); } -struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig); +struct evsel *evsel__clone(struct evsel *orig); int copy_config_terms(struct list_head *dst, struct list_head *src); void free_config_terms(struct list_head *config_terms); @@ -286,14 +288,13 @@ static inline struct evsel *evsel__newtp(const char *sys, const char *name) return evsel__newtp_idx(sys, name, 0, true); } +struct evsel *evsel__get(struct evsel *evsel); +void evsel__put(struct evsel *evsel); + #ifdef HAVE_LIBTRACEEVENT struct tep_event *evsel__tp_format(struct evsel *evsel); #endif -void evsel__init(struct evsel *evsel, struct perf_event_attr *attr, int idx); -void evsel__exit(struct evsel *evsel); -void evsel__delete(struct evsel *evsel); - void evsel__set_priv_destructor(void (*destructor)(void *priv)); struct callchain_param; @@ -400,6 +401,12 @@ static inline char *perf_sample__strval(struct perf_sample *sample, const char * struct tep_format_field; +void *format_field__get_raw_data(struct tep_format_field *field, + struct perf_sample *sample, + bool needs_swap, u16 *len_out); +unsigned long *format_field__get_cpumask(struct tep_format_field *field, + struct perf_sample *sample, + bool needs_swap, u16 *len_out); u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample, bool needs_swap); #ifdef HAVE_LIBTRACEEVENT @@ -534,7 +541,7 @@ for ((_evsel) = list_entry((_leader)->core.node.next, struct evsel, core.node); (_evsel) = list_entry((_evsel)->core.node.next, struct evsel, core.node)) #define for_each_group_member(_evsel, _leader) \ - for_each_group_member_head(_evsel, _leader, &(_leader)->evlist->core.entries) + for_each_group_member_head(_evsel, _leader, &evlist__core((_leader)->evlist)->entries) /* Iterates group WITH the leader. */ #define for_each_group_evsel_head(_evsel, _leader, _head) \ @@ -544,7 +551,7 @@ for ((_evsel) = _leader; \ (_evsel) = list_entry((_evsel)->core.node.next, struct evsel, core.node)) #define for_each_group_evsel(_evsel, _leader) \ - for_each_group_evsel_head(_evsel, _leader, &(_leader)->evlist->core.entries) + for_each_group_evsel_head(_evsel, _leader, &evlist__core((_leader)->evlist)->entries) static inline bool evsel__has_branch_callstack(const struct evsel *evsel) { diff --git a/tools/perf/util/expr.c b/tools/perf/util/expr.c index 232998fef72ba3..8aef3c7418f7c1 100644 --- a/tools/perf/util/expr.c +++ b/tools/perf/util/expr.c @@ -468,7 +468,7 @@ double expr__has_event(const struct expr_parse_ctx *ctx, bool compute_ids, const ret = parse_event(tmp, id) ? 0 : 1; } out: - evlist__delete(tmp); + evlist__put(tmp); return ret; } diff --git a/tools/perf/util/header.c b/tools/perf/util/header.c index 091d8f7f6bd2c9..7db7da090a1e0c 100644 --- a/tools/perf/util/header.c +++ b/tools/perf/util/header.c @@ -385,7 +385,7 @@ static int write_tracing_data(struct feat_fd *ff, return -1; #ifdef HAVE_LIBTRACEEVENT - return read_tracing_data(ff->fd, &evlist->core.entries); + return read_tracing_data(ff->fd, &evlist__core(evlist)->entries); #else pr_err("ERROR: Trying to write tracing data without libtraceevent support.\n"); return -1; @@ -434,8 +434,8 @@ static int write_osrelease(struct feat_fd *ff, struct utsname uts; const char *release = NULL; - if (evlist->session) - release = perf_env__os_release(perf_session__env(evlist->session)); + if (evlist__session(evlist)) + release = perf_env__os_release(perf_session__env(evlist__session(evlist))); if (!release) { int ret = uname(&uts); @@ -452,8 +452,8 @@ static int write_arch(struct feat_fd *ff, struct evlist *evlist) struct utsname uts; const char *arch = NULL; - if (evlist->session) - arch = perf_env__arch(perf_session__env(evlist->session)); + if (evlist__session(evlist)) + arch = perf_env__arch(perf_session__env(evlist__session(evlist))); if (!arch) { int ret = uname(&uts); @@ -469,7 +469,7 @@ static int write_e_machine(struct feat_fd *ff, struct evlist *evlist) { /* e_machine expanded from 16 to 32-bits for alignment. */ uint32_t e_flags; - uint32_t e_machine = perf_session__e_machine(evlist->session, &e_flags); + uint32_t e_machine = perf_session__e_machine(evlist__session(evlist), &e_flags); int ret; ret = do_write(ff, &e_machine, sizeof(e_machine)); @@ -605,7 +605,7 @@ static int write_event_desc(struct feat_fd *ff, u32 nre, nri, sz; int ret; - nre = evlist->core.nr_entries; + nre = evlist__nr_entries(evlist); /* * write number of events @@ -987,7 +987,7 @@ int __weak get_cpuid(char *buffer __maybe_unused, size_t sz __maybe_unused, static int write_cpuid(struct feat_fd *ff, struct evlist *evlist) { - struct perf_cpu cpu = perf_cpu_map__min(evlist->core.all_cpus); + struct perf_cpu cpu = perf_cpu_map__min(evlist__core(evlist)->all_cpus); char buffer[64]; int ret; @@ -1120,7 +1120,8 @@ static int write_bpf_prog_info(struct feat_fd *ff __maybe_unused, struct rb_node *next; int ret = 0; - down_read(&env->bpf_progs.lock); + /* write lock: bpil_addr_to_offs() temporarily mutates info_linear */ + down_write(&env->bpf_progs.lock); ret = do_write(ff, &env->bpf_progs.infos_cnt, sizeof(env->bpf_progs.infos_cnt)); @@ -1150,7 +1151,7 @@ static int write_bpf_prog_info(struct feat_fd *ff __maybe_unused, goto out; } out: - up_read(&env->bpf_progs.lock); + up_write(&env->bpf_progs.lock); return ret; #else pr_err("ERROR: Trying to write bpf_prog_info without libbpf support.\n"); @@ -1420,14 +1421,14 @@ static int write_sample_time(struct feat_fd *ff, struct evlist *evlist) { int ret; + u64 data = evlist__first_sample_time(evlist); - ret = do_write(ff, &evlist->first_sample_time, - sizeof(evlist->first_sample_time)); + ret = do_write(ff, &data, sizeof(data)); if (ret < 0) return ret; - return do_write(ff, &evlist->last_sample_time, - sizeof(evlist->last_sample_time)); + data = evlist__last_sample_time(evlist); + return do_write(ff, &data, sizeof(data)); } @@ -2551,16 +2552,16 @@ static void print_sample_time(struct feat_fd *ff, FILE *fp) session = container_of(ff->ph, struct perf_session, header); - timestamp__scnprintf_usec(session->evlist->first_sample_time, + timestamp__scnprintf_usec(evlist__first_sample_time(session->evlist), time_buf, sizeof(time_buf)); fprintf(fp, "# time of first sample : %s\n", time_buf); - timestamp__scnprintf_usec(session->evlist->last_sample_time, + timestamp__scnprintf_usec(evlist__last_sample_time(session->evlist), time_buf, sizeof(time_buf)); fprintf(fp, "# time of last sample : %s\n", time_buf); - d = (double)(session->evlist->last_sample_time - - session->evlist->first_sample_time) / NSEC_PER_MSEC; + d = (double)(evlist__last_sample_time(session->evlist) - + evlist__first_sample_time(session->evlist)) / NSEC_PER_MSEC; fprintf(fp, "# sample duration : %10.3f ms\n", d); } @@ -3519,8 +3520,8 @@ static int process_sample_time(struct feat_fd *ff, void *data __maybe_unused) if (ret) return -1; - session->evlist->first_sample_time = first_sample_time; - session->evlist->last_sample_time = last_sample_time; + evlist__set_first_sample_time(session->evlist, first_sample_time); + evlist__set_last_sample_time(session->evlist, last_sample_time); return 0; } @@ -4610,7 +4611,7 @@ int perf_session__write_header(struct perf_session *session, /*write_attrs_after_data=*/false); } -size_t perf_session__data_offset(const struct evlist *evlist) +size_t perf_session__data_offset(struct evlist *evlist) { struct evsel *evsel; size_t data_offset; @@ -4619,7 +4620,7 @@ size_t perf_session__data_offset(const struct evlist *evlist) evlist__for_each_entry(evlist, evsel) { data_offset += evsel->core.ids * sizeof(u64); } - data_offset += evlist->core.nr_entries * sizeof(struct perf_file_attr); + data_offset += evlist__nr_entries(evlist) * sizeof(struct perf_file_attr); return data_offset; } @@ -5110,7 +5111,7 @@ int perf_session__read_header(struct perf_session *session) if (session->evlist == NULL) return -ENOMEM; - session->evlist->session = session; + evlist__set_session(session->evlist, session); session->machines.host.env = &header->env; /* @@ -5186,7 +5187,7 @@ int perf_session__read_header(struct perf_session *session) pr_err("Invalid ids section size %" PRIu64 " for attr %d, not aligned to u64\n", f_attr.ids.size, i); err = -EINVAL; - goto out_delete_evlist; + goto out_put_evlist; } /* @@ -5199,7 +5200,7 @@ int perf_session__read_header(struct perf_session *session) pr_err("Invalid ids section size %" PRIu64 " for attr %d, too many IDs\n", f_attr.ids.size, i); err = -EINVAL; - goto out_delete_evlist; + goto out_put_evlist; } /* @@ -5212,19 +5213,19 @@ int perf_session__read_header(struct perf_session *session) pr_err("Invalid ids section for attr %d: offset=%" PRIu64 " size=%" PRIu64 " exceeds file size %" PRIu64 "\n", i, f_attr.ids.offset, f_attr.ids.size, (u64)input_stat.st_size); err = -EINVAL; - goto out_delete_evlist; + goto out_put_evlist; } tmp = lseek(fd, 0, SEEK_CUR); evsel = evsel__new(&f_attr.attr); if (evsel == NULL) - goto out_delete_evlist; + goto out_put_evlist; evsel->needs_swap = header->needs_swap; /* * Do it before so that if perf_evsel__alloc_id fails, this - * entry gets purged too at evlist__delete(). + * entry gets purged too at evlist__put(). */ evlist__add(session->evlist, evsel); @@ -5235,7 +5236,7 @@ int perf_session__read_header(struct perf_session *session) * hattr->ids threads. */ if (perf_evsel__alloc_id(&evsel->core, 1, nr_ids)) - goto out_delete_evlist; + goto out_put_evlist; lseek(fd, f_attr.ids.offset, SEEK_SET); @@ -5243,7 +5244,8 @@ int perf_session__read_header(struct perf_session *session) if (perf_header__getbuffer64(header, fd, &f_id, sizeof(f_id))) goto out_errno; - perf_evlist__id_add(&session->evlist->core, &evsel->core, 0, j, f_id); + perf_evlist__id_add(evlist__core(session->evlist), + &evsel->core, 0, j, f_id); } lseek(fd, tmp, SEEK_SET); @@ -5265,18 +5267,18 @@ int perf_session__read_header(struct perf_session *session) err = perf_header__process_sections(header, fd, &session->tevent, perf_file_section__process); if (err < 0) - goto out_delete_evlist; + goto out_put_evlist; if (evlist__prepare_tracepoint_events(session->evlist, session->tevent.pevent)) { err = -ENOMEM; - goto out_delete_evlist; + goto out_put_evlist; } #else err = perf_header__process_sections(header, fd, NULL, perf_file_section__process); if (err < 0) - goto out_delete_evlist; + goto out_put_evlist; #endif } @@ -5302,8 +5304,8 @@ int perf_session__read_header(struct perf_session *session) out_errno: return -errno; -out_delete_evlist: - evlist__delete(session->evlist); +out_put_evlist: + evlist__put(session->evlist); session->evlist = NULL; return err; } @@ -5607,7 +5609,7 @@ int perf_event__process_attr(const struct perf_tool *tool __maybe_unused, */ ids = (void *)&event->attr.attr + attr_size; for (i = 0; i < n_ids; i++) { - perf_evlist__id_add(&evlist->core, &evsel->core, 0, i, ids[i]); + perf_evlist__id_add(evlist__core(evlist), &evsel->core, 0, i, ids[i]); } return 0; diff --git a/tools/perf/util/header.h b/tools/perf/util/header.h index 86b1a72026d3f3..5e03f884b7cc05 100644 --- a/tools/perf/util/header.h +++ b/tools/perf/util/header.h @@ -158,7 +158,7 @@ int perf_session__inject_header(struct perf_session *session, struct feat_copier *fc, bool write_attrs_after_data); -size_t perf_session__data_offset(const struct evlist *evlist); +size_t perf_session__data_offset(struct evlist *evlist); void perf_header__set_feat(struct perf_header *header, int feat); void perf_header__clear_feat(struct perf_header *header, int feat); diff --git a/tools/perf/util/intel-bts.c b/tools/perf/util/intel-bts.c index 0b18ebd13f7c84..02df3e46048912 100644 --- a/tools/perf/util/intel-bts.c +++ b/tools/perf/util/intel-bts.c @@ -830,7 +830,7 @@ int intel_bts_process_auxtrace_info(union perf_event *event, struct perf_session *session) { struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; - size_t min_sz = sizeof(u64) * INTEL_BTS_SNAPSHOT_MODE; + size_t min_sz = sizeof(u64) * (INTEL_BTS_SNAPSHOT_MODE + 1); struct intel_bts *bts; int err; diff --git a/tools/perf/util/intel-pt.c b/tools/perf/util/intel-pt.c index 56a9e439f5f8fe..806cd173a00676 100644 --- a/tools/perf/util/intel-pt.c +++ b/tools/perf/util/intel-pt.c @@ -4430,7 +4430,7 @@ int intel_pt_process_auxtrace_info(union perf_event *event, struct perf_session *session) { struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; - size_t min_sz = sizeof(u64) * INTEL_PT_PER_CPU_MMAPS; + size_t min_sz = sizeof(u64) * (INTEL_PT_PER_CPU_MMAPS + 1); struct intel_pt *pt; void *info_end; __u64 *info; diff --git a/tools/perf/util/intel-tpebs.c b/tools/perf/util/intel-tpebs.c index bc3b79bfa01a73..b41171b5df77d0 100644 --- a/tools/perf/util/intel-tpebs.c +++ b/tools/perf/util/intel-tpebs.c @@ -98,8 +98,9 @@ static int evsel__tpebs_start_perf_record(struct evsel *evsel) record_argv[i++] = "-o"; record_argv[i++] = PERF_DATA; - if (!perf_cpu_map__is_any_cpu_or_is_empty(evsel->evlist->core.user_requested_cpus)) { - cpu_map__snprint(evsel->evlist->core.user_requested_cpus, cpumap_buf, + if (!perf_cpu_map__is_any_cpu_or_is_empty( + evlist__core(evsel->evlist)->user_requested_cpus)) { + cpu_map__snprint(evlist__core(evsel->evlist)->user_requested_cpus, cpumap_buf, sizeof(cpumap_buf)); record_argv[i++] = "-C"; record_argv[i++] = cpumap_buf; @@ -176,7 +177,7 @@ static bool should_ignore_sample(const struct perf_sample *sample, const struct if (t->evsel->evlist == NULL) return true; - workload_pid = t->evsel->evlist->workload.pid; + workload_pid = evlist__workload_pid(t->evsel->evlist); if (workload_pid < 0 || workload_pid == sample_pid) return false; diff --git a/tools/perf/util/iostat.c b/tools/perf/util/iostat.c index b770bd473af710..c9d5028a47f39e 100644 --- a/tools/perf/util/iostat.c +++ b/tools/perf/util/iostat.c @@ -4,7 +4,7 @@ enum iostat_mode_t iostat_mode = IOSTAT_NONE; -__weak int iostat_prepare(struct evlist *evlist __maybe_unused, +__weak int iostat_prepare(struct evlist **evlist __maybe_unused, struct perf_stat_config *config __maybe_unused) { return -1; diff --git a/tools/perf/util/iostat.h b/tools/perf/util/iostat.h index a4e7299c5c2fbc..df8a241fbc32a0 100644 --- a/tools/perf/util/iostat.h +++ b/tools/perf/util/iostat.h @@ -30,7 +30,7 @@ extern enum iostat_mode_t iostat_mode; typedef void (*iostat_print_counter_t)(struct perf_stat_config *, struct evsel *, void *); -int iostat_prepare(struct evlist *evlist, struct perf_stat_config *config); +int iostat_prepare(struct evlist **evlist, struct perf_stat_config *config); int iostat_parse(const struct option *opt, const char *str, int unset __maybe_unused); void iostat_list(struct evlist *evlist, struct perf_stat_config *config); diff --git a/tools/perf/util/kvm-stat.h b/tools/perf/util/kvm-stat.h index cdbd921a555f4e..104f59262ec90c 100644 --- a/tools/perf/util/kvm-stat.h +++ b/tools/perf/util/kvm-stat.h @@ -69,7 +69,7 @@ struct kvm_events_ops { }; struct exit_reasons_table { - unsigned long exit_code; + u64 exit_code; const char *reason; }; diff --git a/tools/perf/util/libbfd.c b/tools/perf/util/libbfd.c index c1c12308cc12ff..118f973298ea84 100644 --- a/tools/perf/util/libbfd.c +++ b/tools/perf/util/libbfd.c @@ -15,6 +15,7 @@ #ifdef HAVE_LIBBPF_SUPPORT #include #include +#include #endif #include #include @@ -510,7 +511,7 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, char tpath[PATH_MAX]; size_t buf_size; int nr_skip = 0; - char *buf; + char *buf = NULL; bfd *bfdf; int ret; FILE *s; @@ -552,6 +553,11 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, info_linear = info_node->info_linear; sub_id = dso__bpf_prog(dso)->sub_id; + /* jited_prog_insns is only valid if bpil_offs_to_addr() converted it */ + if (!(info_linear->arrays & (1UL << PERF_BPIL_JITED_INSNS))) { + ret = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; + goto out; + } info.buffer = (void *)(uintptr_t)(info_linear->info.jited_prog_insns); info.buffer_length = info_linear->info.jited_prog_len; @@ -581,6 +587,12 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, if (disassemble == NULL) abort(); + /* jited_ksyms is only valid if bpil_offs_to_addr() converted it */ + if (!(info_linear->arrays & (1UL << PERF_BPIL_JITED_KSYMS))) { + ret = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; + goto out; + } + fflush(s); do { const struct bpf_line_info *linfo = NULL; @@ -609,7 +621,7 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, if (!annotate_opts.hide_src_code && srcline) { args->offset = -1; - args->line = strdup(srcline); + args->line = (char *)srcline; args->line_nr = 0; args->fileloc = NULL; args->ms->sym = sym; @@ -634,9 +646,12 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, ret = 0; out: - free(prog_linfo); + bpf_prog_linfo__free(prog_linfo); btf__free(btf); - fclose(s); + if (s) { + fclose(s); + free(buf); + } bfd_close(bfdf); return ret; #else diff --git a/tools/perf/util/libdw.c b/tools/perf/util/libdw.c index d5d2958902c0a2..4ca7e7e4fbe93c 100644 --- a/tools/perf/util/libdw.c +++ b/tools/perf/util/libdw.c @@ -82,13 +82,39 @@ struct libdw_a2l_cb_args { static int libdw_a2l_cb(Dwarf_Die *die, void *_args) { struct libdw_a2l_cb_args *args = _args; - struct symbol *inline_sym = new_inline_sym(args->dso, args->sym, die_name(die)); const char *call_fname = die_get_call_file(die); int call_lineno = die_get_call_lineno(die); char *call_srcline = srcline__unknown; - - if (!inline_sym) - goto abort_enomem; + struct symbol *inline_sym; + + if (dwarf_tag(die) == DW_TAG_subprogram && args->sym) { + /* + * cu_walk_functions_at() opens the walk with the + * containing DW_TAG_subprogram DIE (the non-inlined outer + * function). That's just the base symbol -- use it + * directly. Avoids a fragile name-vs-name compare in + * new_inline_sym() that misfires when GCC IPA passes + * (.isra/.constprop/.part/.cold) rename the ELF symbol + * while DWARF keeps the pre-clone linkage name, which + * left the outer frame spuriously tagged "(inlined)". + */ + inline_sym = args->sym; + } else { + /* + * Prefer DW_AT_linkage_name so C++ inline frames keep + * their namespace/class qualification. new_inline_sym() + * runs the name through dso__demangle_sym(), so the + * mangled linkage name is turned back into + * "Namespace::Class::method". Fall back to DW_AT_name + * (unqualified) when no linkage name is present, e.g. + * for C code or extern "C" functions. + */ + const char *funcname = die_get_linkage_name(die) ?: die_name(die); + + inline_sym = new_inline_sym(args->dso, args->sym, funcname); + if (!inline_sym) + goto abort_enomem; + } /* Assign caller information to the parent. */ if (call_fname) diff --git a/tools/perf/util/machine.c b/tools/perf/util/machine.c index 0d2ebf6a84bcf8..f86b3b7df742e0 100644 --- a/tools/perf/util/machine.c +++ b/tools/perf/util/machine.c @@ -333,16 +333,22 @@ struct machine *machines__findnew(struct machines *machines, pid_t pid) if ((pid != HOST_KERNEL_ID) && (pid != DEFAULT_GUEST_KERNEL_ID) && (symbol_conf.guestmount)) { - snprintf(path, sizeof(path), "%s/%d", symbol_conf.guestmount, pid); + if (snprintf(path, sizeof(path), "%s/%d", + symbol_conf.guestmount, pid) >= (int)sizeof(path)) { + pr_err("Guest path too long for pid %d\n", pid); + machine = NULL; + goto out; + } if (access(path, R_OK)) { static struct strlist *seen; if (!seen) seen = strlist__new(NULL, NULL); - if (!strlist__has_entry(seen, path)) { + if (!seen || !strlist__has_entry(seen, path)) { pr_err("Can't access file %s\n", path); - strlist__add(seen, path); + if (seen) + strlist__add(seen, path); } machine = NULL; goto out; @@ -1250,27 +1256,35 @@ int machines__create_guest_kernel_maps(struct machines *machines) for (i = 0; i < items; i++) { if (!isdigit(namelist[i]->d_name[0])) { /* Filter out . and .. */ + free(namelist[i]); continue; } + errno = 0; pid = (pid_t)strtol(namelist[i]->d_name, &endp, 10); if ((*endp != '\0') || (endp == namelist[i]->d_name) || (errno == ERANGE)) { pr_debug("invalid directory (%s). Skipping.\n", namelist[i]->d_name); + free(namelist[i]); + continue; + } + if (snprintf(path, sizeof(path), "%s/%s/proc/kallsyms", + symbol_conf.guestmount, + namelist[i]->d_name) >= (int)sizeof(path)) { + pr_debug("Guest kallsyms path too long for %s. Skipping.\n", + namelist[i]->d_name); + free(namelist[i]); continue; } - snprintf(path, sizeof(path), "%s/%s/proc/kallsyms", - symbol_conf.guestmount, - namelist[i]->d_name); - ret = access(path, R_OK); - if (ret) { + if (access(path, R_OK)) { pr_debug("Can't access file %s\n", path); - goto failure; + free(namelist[i]); + continue; } machines__create_kernel_maps(machines, pid); + free(namelist[i]); } -failure: free(namelist); } @@ -1411,8 +1425,10 @@ static int maps__set_modules_path_dir(struct maps *maps, char *path, size_t path return -1; } /* Bounds check, should never happen. */ - if (root_len >= path_size) - return -1; + if (root_len >= path_size) { + ret = -1; + goto out; + } path[root_len++] = '/'; while ((dent = io_dir__readdir(&iod)) != NULL) { if (io_dir__is_dir(&iod, dent)) { @@ -1921,7 +1937,8 @@ int machine__process_fork_event(struct machine *machine, union perf_event *event * (fork) event that would have removed the thread was lost. Assume the * latter case and continue on as best we can. */ - if (thread__pid(parent) != (pid_t)event->fork.ppid) { + if (parent != NULL && + thread__pid(parent) != (pid_t)event->fork.ppid) { dump_printf("removing erroneous parent thread %d/%d\n", thread__pid(parent), thread__tid(parent)); machine__remove_thread(machine, parent); diff --git a/tools/perf/util/map.h b/tools/perf/util/map.h index 979b3e11b9bcf2..fb0279810ae99e 100644 --- a/tools/perf/util/map.h +++ b/tools/perf/util/map.h @@ -2,14 +2,13 @@ #ifndef __PERF_MAP_H #define __PERF_MAP_H -#include -#include -#include -#include +#include #include #include -#include + +#include #include + #include struct dso; diff --git a/tools/perf/util/metricgroup.c b/tools/perf/util/metricgroup.c index c2ce3e53aaee76..5a60cb95e31c7d 100644 --- a/tools/perf/util/metricgroup.c +++ b/tools/perf/util/metricgroup.c @@ -214,7 +214,7 @@ static void metric__free(struct metric *m) zfree(&m->metric_refs); expr__ctx_free(m->pctx); zfree(&m->modifier); - evlist__delete(m->evlist); + evlist__put(m->evlist); free(m); } @@ -1262,7 +1262,8 @@ static int parse_ids(bool metric_no_merge, bool fake_pmu, struct expr_parse_ctx *ids, const char *modifier, bool group_events, const bool tool_events[TOOL_PMU__EVENT_MAX], struct evlist **out_evlist, - const char *filter_pmu) + const char *filter_pmu, + bool cputype_filter) { struct parse_events_error parse_error; struct evlist *parsed_evlist; @@ -1317,7 +1318,9 @@ static int parse_ids(bool metric_no_merge, bool fake_pmu, pr_debug("Parsing metric events '%s'\n", events.buf); parse_events_error__init(&parse_error); ret = __parse_events(parsed_evlist, events.buf, filter_pmu, - &parse_error, fake_pmu, /*warn_if_reordered=*/false, + cputype_filter, + &parse_error, fake_pmu, + /*warn_if_reordered=*/false, /*fake_tp=*/false); if (ret) { parse_events_error__print(&parse_error, events.buf); @@ -1331,7 +1334,7 @@ static int parse_ids(bool metric_no_merge, bool fake_pmu, parsed_evlist = NULL; err_out: parse_events_error__exit(&parse_error); - evlist__delete(parsed_evlist); + evlist__put(parsed_evlist); strbuf_release(&events); return ret; } @@ -1382,7 +1385,7 @@ static struct evsel *pick_display_evsel(struct list_head *metric_list, } static int parse_groups(struct evlist *perf_evlist, - const char *pmu, const char *str, + const char *pmu, bool cputype_filter, const char *str, bool metric_no_group, bool metric_no_merge, bool metric_no_threshold, @@ -1420,7 +1423,8 @@ static int parse_groups(struct evlist *perf_evlist, /*group_events=*/false, tool_events, &combined_evlist, - (pmu && strcmp(pmu, "all") == 0) ? NULL : pmu); + (pmu && strcmp(pmu, "all") == 0) ? NULL : pmu, + cputype_filter); } if (combined) expr__ctx_free(combined); @@ -1476,7 +1480,8 @@ static int parse_groups(struct evlist *perf_evlist, if (!metric_evlist) { ret = parse_ids(metric_no_merge, fake_pmu, m->pctx, m->modifier, m->group_events, tool_events, &m->evlist, - (pmu && strcmp(pmu, "all") == 0) ? NULL : pmu); + (pmu && strcmp(pmu, "all") == 0) ? NULL : pmu, + cputype_filter); if (ret) goto out; @@ -1490,7 +1495,7 @@ static int parse_groups(struct evlist *perf_evlist, goto out; } - me = metricgroup__lookup(&perf_evlist->metric_events, + me = metricgroup__lookup(evlist__metric_events(perf_evlist), pick_display_evsel(&metric_list, metric_events), /*create=*/true); @@ -1541,22 +1546,26 @@ static int parse_groups(struct evlist *perf_evlist, if (combined_evlist) { - evlist__splice_list_tail(perf_evlist, &combined_evlist->core.entries); - evlist__delete(combined_evlist); + evlist__splice_list_tail(perf_evlist, &evlist__core(combined_evlist)->entries); + evlist__put(combined_evlist); + combined_evlist = NULL; } list_for_each_entry(m, &metric_list, nd) { if (m->evlist) - evlist__splice_list_tail(perf_evlist, &m->evlist->core.entries); + evlist__splice_list_tail(perf_evlist, &evlist__core(m->evlist)->entries); } out: + if (combined_evlist) + evlist__put(combined_evlist); metricgroup__free_metrics(&metric_list); return ret; } int metricgroup__parse_groups(struct evlist *perf_evlist, const char *pmu, + bool cputype_filter, const char *str, bool metric_no_group, bool metric_no_merge, @@ -1570,16 +1579,17 @@ int metricgroup__parse_groups(struct evlist *perf_evlist, if (hardware_aware_grouping) pr_debug("Use hardware aware grouping instead of traditional metric grouping method\n"); - return parse_groups(perf_evlist, pmu, str, metric_no_group, metric_no_merge, + return parse_groups(perf_evlist, pmu, cputype_filter, str, metric_no_group, metric_no_merge, metric_no_threshold, user_requested_cpu_list, system_wide, /*fake_pmu=*/false, table); } int metricgroup__parse_groups_test(struct evlist *evlist, const struct pmu_metrics_table *table, - const char *str) + const char *str, + bool cputype_filter) { - return parse_groups(evlist, "all", str, + return parse_groups(evlist, "all", cputype_filter, str, /*metric_no_group=*/false, /*metric_no_merge=*/false, /*metric_no_threshold=*/false, @@ -1683,8 +1693,10 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp, new_expr->metric_expr = old_expr->metric_expr; new_expr->metric_threshold = old_expr->metric_threshold; new_expr->metric_name = strdup(old_expr->metric_name); - if (!new_expr->metric_name) + if (!new_expr->metric_name) { + free(new_expr); return -ENOMEM; + } new_expr->metric_unit = old_expr->metric_unit; new_expr->runtime = old_expr->runtime; @@ -1697,6 +1709,7 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp, alloc_size = sizeof(*new_expr->metric_refs); new_expr->metric_refs = calloc(nr + 1, alloc_size); if (!new_expr->metric_refs) { + zfree(&new_expr->metric_name); free(new_expr); return -ENOMEM; } @@ -1713,6 +1726,7 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp, alloc_size = sizeof(*new_expr->metric_events); new_expr->metric_events = calloc(nr + 1, alloc_size); if (!new_expr->metric_events) { + zfree(&new_expr->metric_name); zfree(&new_expr->metric_refs); free(new_expr); return -ENOMEM; @@ -1723,6 +1737,7 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp, evsel = old_expr->metric_events[idx]; evsel = evlist__find_evsel(evlist, evsel->core.idx); if (evsel == NULL) { + zfree(&new_expr->metric_name); zfree(&new_expr->metric_events); zfree(&new_expr->metric_refs); free(new_expr); diff --git a/tools/perf/util/metricgroup.h b/tools/perf/util/metricgroup.h index 4be6bfc13c467d..6a66f14dd01bd9 100644 --- a/tools/perf/util/metricgroup.h +++ b/tools/perf/util/metricgroup.h @@ -71,6 +71,7 @@ struct metric_event *metricgroup__lookup(struct rblist *metric_events, bool create); int metricgroup__parse_groups(struct evlist *perf_evlist, const char *pmu, + bool cputype_filter, const char *str, bool metric_no_group, bool metric_no_merge, @@ -80,7 +81,8 @@ int metricgroup__parse_groups(struct evlist *perf_evlist, bool hardware_aware_grouping); int metricgroup__parse_groups_test(struct evlist *evlist, const struct pmu_metrics_table *table, - const char *str); + const char *str, + bool cputype_filter); int metricgroup__for_each_metric(const struct pmu_metrics_table *table, pmu_metric_iter_fn fn, void *data); diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c index 943569e82b82f9..cc7ad331a49f34 100644 --- a/tools/perf/util/parse-events.c +++ b/tools/perf/util/parse-events.c @@ -429,6 +429,9 @@ bool parse_events__filter_pmu(const struct parse_events_state *parse_state, if (parse_state->pmu_filter == NULL) return false; + if (parse_state->cputype_filter && !pmu->is_core) + return false; + return perf_pmu__wildcard_match(pmu, parse_state->pmu_filter) == 0; } @@ -2288,18 +2291,20 @@ static int parse_events__sort_events_and_fix_groups(struct list_head *list) return (idx_changed || num_leaders != orig_num_leaders) ? 1 : 0; } -int __parse_events(struct evlist *evlist, const char *str, const char *pmu_filter, +int __parse_events(struct evlist *evlist, const char *str, + const char *pmu_filter, bool cputype_filter, struct parse_events_error *err, bool fake_pmu, bool warn_if_reordered, bool fake_tp) { struct parse_events_state parse_state = { - .list = LIST_HEAD_INIT(parse_state.list), - .idx = evlist->core.nr_entries, - .error = err, - .stoken = PE_START_EVENTS, - .fake_pmu = fake_pmu, - .fake_tp = fake_tp, - .pmu_filter = pmu_filter, + .list = LIST_HEAD_INIT(parse_state.list), + .idx = evlist__nr_entries(evlist), + .error = err, + .stoken = PE_START_EVENTS, + .fake_pmu = fake_pmu, + .fake_tp = fake_tp, + .pmu_filter = pmu_filter, + .cputype_filter = cputype_filter, .match_legacy_cache_terms = true, }; int ret, ret2; @@ -2312,15 +2317,15 @@ int __parse_events(struct evlist *evlist, const char *str, const char *pmu_filte } ret2 = parse_events__sort_events_and_fix_groups(&parse_state.list); - if (ret2 < 0) - return ret; + if (ret2 < 0 && !ret) + ret = ret2; /* * Add list to the evlist even with errors to allow callers to clean up. */ evlist__splice_list_tail(evlist, &parse_state.list); - if (ret2 && warn_if_reordered && !parse_state.wild_card_pmus) { + if (ret2 > 0 && warn_if_reordered && !parse_state.wild_card_pmus) { evlist__uniquify_evsel_names(evlist, &stat_config); pr_warning("WARNING: events were regrouped to match PMUs\n"); @@ -2343,7 +2348,7 @@ int __parse_events(struct evlist *evlist, const char *str, const char *pmu_filte /* * There are 2 users - builtin-record and builtin-test objects. - * Both call evlist__delete in case of error, so we dont + * Both call evlist__put in case of error, so we dont * need to bother. */ return ret; @@ -2518,8 +2523,9 @@ int parse_events_option(const struct option *opt, const char *str, int ret; parse_events_error__init(&err); - ret = __parse_events(*args->evlistp, str, args->pmu_filter, &err, - /*fake_pmu=*/false, /*warn_if_reordered=*/true, + ret = __parse_events(*args->evlistp, str, args->pmu_filter, + args->cputype_filter, &err, /*fake_pmu=*/false, + /*warn_if_reordered=*/true, /*fake_tp=*/false); if (ret) { @@ -2546,7 +2552,7 @@ int parse_events_option_new_evlist(const struct option *opt, const char *str, in } ret = parse_events_option(opt, str, unset); if (ret) { - evlist__delete(*args->evlistp); + evlist__put(*args->evlistp); *args->evlistp = NULL; } @@ -2568,7 +2574,7 @@ foreach_evsel_in_last_glob(struct evlist *evlist, * * So no need to WARN here, let *func do this. */ - if (evlist->core.nr_entries > 0) + if (evlist__nr_entries(evlist) > 0) last = evlist__last(evlist); do { @@ -2578,7 +2584,7 @@ foreach_evsel_in_last_glob(struct evlist *evlist, if (!last) return 0; - if (last->core.node.prev == &evlist->core.entries) + if (last->core.node.prev == &evlist__core(evlist)->entries) return 0; last = list_entry(last->core.node.prev, struct evsel, core.node); } while (!last->cmdline_group_boundary); diff --git a/tools/perf/util/parse-events.h b/tools/perf/util/parse-events.h index 3577ab2137304d..b14c832b03a1e4 100644 --- a/tools/perf/util/parse-events.h +++ b/tools/perf/util/parse-events.h @@ -26,20 +26,23 @@ const char *event_type(size_t type); struct parse_events_option_args { struct evlist **evlistp; const char *pmu_filter; + bool cputype_filter; }; int parse_events_option(const struct option *opt, const char *str, int unset); int parse_events_option_new_evlist(const struct option *opt, const char *str, int unset); -__attribute__((nonnull(1, 2, 4))) -int __parse_events(struct evlist *evlist, const char *str, const char *pmu_filter, - struct parse_events_error *error, bool fake_pmu, - bool warn_if_reordered, bool fake_tp); +__attribute__((nonnull(1, 2, 5))) int +__parse_events(struct evlist *evlist, const char *str, const char *pmu_filter, + bool cputype_filter, struct parse_events_error *error, + bool fake_pmu, bool warn_if_reordered, bool fake_tp); __attribute__((nonnull(1, 2, 3))) static inline int parse_events(struct evlist *evlist, const char *str, struct parse_events_error *err) { - return __parse_events(evlist, str, /*pmu_filter=*/NULL, err, /*fake_pmu=*/false, - /*warn_if_reordered=*/true, /*fake_tp=*/false); + return __parse_events(evlist, str, /*pmu_filter=*/NULL, + /*cputype_filter=*/false, err, /*fake_pmu=*/false, + /*warn_if_reordered=*/true, + /*fake_tp=*/false); } int parse_event(struct evlist *evlist, const char *str); @@ -161,6 +164,8 @@ struct parse_events_state { bool fake_tp; /* If non-null, when wildcard matching only match the given PMU. */ const char *pmu_filter; + /* If true, the pmu_filter was set by --cputype option. */ + bool cputype_filter; /* Should PE_LEGACY_NAME tokens be generated for config terms? */ bool match_legacy_cache_terms; /* Were multiple PMUs scanned to find events? */ diff --git a/tools/perf/util/parse-events.y b/tools/perf/util/parse-events.y index c194de5ec1ec7c..b531b1f0ceb33a 100644 --- a/tools/perf/util/parse-events.y +++ b/tools/perf/util/parse-events.y @@ -47,7 +47,7 @@ static void free_list_evsel(struct list_head* list_evsel) list_for_each_entry_safe(evsel, tmp, list_evsel, core.node) { list_del_init(&evsel->core.node); - evsel__delete(evsel); + evsel__put(evsel); } free(list_evsel); } diff --git a/tools/perf/util/perf_api_probe.c b/tools/perf/util/perf_api_probe.c index 6ecf38314f01c6..f61c4ec52827b3 100644 --- a/tools/perf/util/perf_api_probe.c +++ b/tools/perf/util/perf_api_probe.c @@ -1,14 +1,18 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#include "perf_api_probe.h" -#include "perf-sys.h" -#include "util/cloexec.h" -#include "util/evlist.h" -#include "util/evsel.h" -#include "util/parse-events.h" -#include "util/perf_api_probe.h" -#include #include +#include + +#include "cloexec.h" +#include "evlist.h" +#include "evsel.h" +#include "parse-events.h" +#include "perf-sys.h" +#include "pmu.h" +#include "pmus.h" + typedef void (*setup_probe_fn_t)(struct evsel *evsel); static int perf_do_probe_api(setup_probe_fn_t fn, struct perf_cpu cpu, const char *str) @@ -53,7 +57,7 @@ static int perf_do_probe_api(setup_probe_fn_t fn, struct perf_cpu cpu, const cha err = 0; out_delete: - evlist__delete(evlist); + evlist__put(evlist); return err; } diff --git a/tools/perf/util/pfm.c b/tools/perf/util/pfm.c index d9043f4afbe7b1..f80d6b0df47ae6 100644 --- a/tools/perf/util/pfm.c +++ b/tools/perf/util/pfm.c @@ -85,7 +85,7 @@ int parse_libpfm_events_option(const struct option *opt, const char *str, } pmu = perf_pmus__find_by_type((unsigned int)attr.type); - evsel = parse_events__add_event(evlist->core.nr_entries, + evsel = parse_events__add_event(evlist__nr_entries(evlist), &attr, q, /*metric_id=*/NULL, pmu); if (evsel == NULL) @@ -159,7 +159,7 @@ static bool is_libpfm_event_supported(const char *name, struct perf_cpu_map *cpu result = false; evsel__close(evsel); - evsel__delete(evsel); + evsel__put(evsel); return result; } diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index a550f030b85df3..836e3b5615cd84 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -2660,8 +2660,12 @@ bool perf_pmu__wildcard_match(const struct perf_pmu *pmu, const char *wildcard_t pmu->name, pmu->alias_name, }; - bool need_fnmatch = strisglob(wildcard_to_match); + bool need_fnmatch; + if (pmu->is_core && !strcmp(wildcard_to_match, "default_core")) + return true; + + need_fnmatch = strisglob(wildcard_to_match); if (!strncmp(wildcard_to_match, "uncore_", 7)) wildcard_to_match += 7; diff --git a/tools/perf/util/pmus.c b/tools/perf/util/pmus.c index 5e3f571450fe7f..e0a4cb2428ca40 100644 --- a/tools/perf/util/pmus.c +++ b/tools/perf/util/pmus.c @@ -150,6 +150,8 @@ struct perf_pmu *perf_pmus__find(const char *name) bool core_pmu; unsigned int to_read_pmus = 0; + if (!strcmp(name, "default_core")) + return perf_pmus__find_core_pmu(); /* * Once PMU is loaded it stays in the list, * so we keep us from multiple reading/parsing diff --git a/tools/perf/util/print-events.c b/tools/perf/util/print-events.c index cb27e2898aa055..0242243681b6b6 100644 --- a/tools/perf/util/print-events.c +++ b/tools/perf/util/print-events.c @@ -174,7 +174,7 @@ bool is_event_supported(u8 type, u64 config) } evsel__close(evsel); - evsel__delete(evsel); + evsel__put(evsel); } perf_thread_map__put(tmap); diff --git a/tools/perf/util/python.c b/tools/perf/util/python.c index cc1019d29a5d01..b6c4b3de5b4d20 100644 --- a/tools/perf/util/python.c +++ b/tools/perf/util/python.c @@ -1,30 +1,37 @@ // SPDX-License-Identifier: GPL-2.0 +#define PY_SSIZE_T_CLEAN #include -#include + #include -#include + #include +#include + +#include #include -#ifdef HAVE_LIBTRACEEVENT -#include -#endif #include +#include + #include "callchain.h" #include "counts.h" +#include "event.h" #include "evlist.h" #include "evsel.h" -#include "event.h" #include "expr.h" +#include "metricgroup.h" +#include "mmap.h" +#include "pmus.h" #include "print_binary.h" #include "record.h" #include "strbuf.h" #include "thread_map.h" #include "tp_pmu.h" #include "trace-event.h" -#include "metricgroup.h" -#include "mmap.h" #include "util/sample.h" -#include + +#ifdef HAVE_LIBTRACEEVENT +#include +#endif PyMODINIT_FUNC PyInit_perf(void); @@ -38,6 +45,22 @@ PyMODINIT_FUNC PyInit_perf(void); offsetof(struct pyrf_event, sample) + offsetof(struct perf_sample, member), \ 0, help } +#define CHECK_INITIALIZED(ptr, msg) \ + do { \ + if (!(ptr)) { \ + PyErr_SetString(PyExc_ValueError, msg " not initialized"); \ + return NULL; \ + } \ + } while (0) + +#define CHECK_INITIALIZED_INT(ptr, msg) \ + do { \ + if (!(ptr)) { \ + PyErr_SetString(PyExc_ValueError, msg " not initialized"); \ + return -1; \ + } \ + } while (0) + struct pyrf_event { PyObject_HEAD struct evsel *evsel; @@ -271,8 +294,9 @@ static PyMemberDef pyrf_sample_event__members[] = { static void pyrf_sample_event__delete(struct pyrf_event *pevent) { + evsel__put(pevent->evsel); perf_sample__exit(&pevent->sample); - Py_TYPE(pevent)->tp_free((PyObject*)pevent); + Py_TYPE(pevent)->tp_free((PyObject *)pevent); } static PyObject *pyrf_sample_event__repr(const struct pyrf_event *pevent) @@ -503,8 +527,10 @@ static PyObject *pyrf_event__new(const union perf_event *event) ptype = pyrf_event__type[event->header.type]; pevent = PyObject_New(struct pyrf_event, ptype); - if (pevent != NULL) + if (pevent != NULL) { memcpy(&pevent->event, event, event->header.size); + pevent->evsel = NULL; + } return (PyObject *)pevent; } @@ -872,11 +898,11 @@ static void pyrf_counts_values__delete(struct pyrf_counts_values *pcounts_values 0, help } static PyMemberDef pyrf_counts_values_members[] = { - counts_values_member_def(val, T_ULONG, "Value of event"), - counts_values_member_def(ena, T_ULONG, "Time for which enabled"), - counts_values_member_def(run, T_ULONG, "Time for which running"), - counts_values_member_def(id, T_ULONG, "Unique ID for an event"), - counts_values_member_def(lost, T_ULONG, "Num of lost samples"), + counts_values_member_def(val, T_ULONGLONG, "Value of event"), + counts_values_member_def(ena, T_ULONGLONG, "Time for which enabled"), + counts_values_member_def(run, T_ULONGLONG, "Time for which running"), + counts_values_member_def(id, T_ULONGLONG, "Unique ID for an event"), + counts_values_member_def(lost, T_ULONGLONG, "Num of lost samples"), { .name = NULL, }, }; @@ -886,8 +912,15 @@ static PyObject *pyrf_counts_values_get_values(struct pyrf_counts_values *self, if (!vals) return NULL; - for (int i = 0; i < 5; i++) - PyList_SetItem(vals, i, PyLong_FromLong(self->values.values[i])); + for (int i = 0; i < 5; i++) { + PyObject *val = PyLong_FromUnsignedLongLong(self->values.values[i]); + + if (!val) { + Py_DECREF(vals); + return NULL; + } + PyList_SetItem(vals, i, val); + } return vals; } @@ -898,19 +931,34 @@ static int pyrf_counts_values_set_values(struct pyrf_counts_values *self, PyObje Py_ssize_t size; PyObject *item = NULL; + if (list == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete attribute"); + return -1; + } + if (!PyList_Check(list)) { PyErr_SetString(PyExc_TypeError, "Value assigned must be a list"); return -1; } size = PyList_Size(list); + if (size != 5) { + PyErr_SetString(PyExc_ValueError, "List must have exactly 5 entries"); + return -1; + } + for (Py_ssize_t i = 0; i < size; i++) { + unsigned long long val; + item = PyList_GetItem(list, i); if (!PyLong_Check(item)) { PyErr_SetString(PyExc_TypeError, "List members should be numbers"); return -1; } - self->values.values[i] = PyLong_AsLong(item); + val = PyLong_AsUnsignedLongLong(item); + if (val == (unsigned long long)-1 && PyErr_Occurred()) + return -1; + self->values.values[i] = val; } return 0; @@ -942,7 +990,7 @@ static int pyrf_counts_values__setup_types(void) struct pyrf_evsel { PyObject_HEAD - struct evsel evsel; + struct evsel *evsel; }; static int pyrf_evsel__init(struct pyrf_evsel *pevsel, @@ -983,6 +1031,7 @@ static int pyrf_evsel__init(struct pyrf_evsel *pevsel, "bp_type", "bp_addr", "bp_len", + "idx", NULL }; u64 sample_period = 0; @@ -1004,11 +1053,11 @@ static int pyrf_evsel__init(struct pyrf_evsel *pevsel, watermark = 0, precise_ip = 0, mmap_data = 0, - sample_id_all = 1; - int idx = 0; + sample_id_all = 1, + idx = 0; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "|iKiKKiiiiiiiiiiiiiiiiiiiiiiKK", kwlist, + "|iKiKKiiiiiiiiiiiiiiiiiiiiiiKKi", kwlist, &attr.type, &attr.config, &attr.sample_freq, &sample_period, &attr.sample_type, &attr.read_format, &disabled, &inherit, @@ -1050,35 +1099,52 @@ static int pyrf_evsel__init(struct pyrf_evsel *pevsel, attr.sample_id_all = sample_id_all; attr.size = sizeof(attr); - evsel__init(&pevsel->evsel, &attr, idx); + evsel__put(pevsel->evsel); + pevsel->evsel = evsel__new(&attr); + if (!pevsel->evsel) { + PyErr_NoMemory(); + return -1; + } return 0; } static void pyrf_evsel__delete(struct pyrf_evsel *pevsel) { - evsel__exit(&pevsel->evsel); + evsel__put(pevsel->evsel); Py_TYPE(pevsel)->tp_free((PyObject*)pevsel); } static PyObject *pyrf_evsel__open(struct pyrf_evsel *pevsel, PyObject *args, PyObject *kwargs) { - struct evsel *evsel = &pevsel->evsel; + struct evsel *evsel = pevsel->evsel; struct perf_cpu_map *cpus = NULL; struct perf_thread_map *threads = NULL; PyObject *pcpus = NULL, *pthreads = NULL; int group = 0, inherit = 0; static char *kwlist[] = { "cpus", "threads", "group", "inherit", NULL }; + CHECK_INITIALIZED(evsel, "evsel"); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|OOii", kwlist, &pcpus, &pthreads, &group, &inherit)) return NULL; - if (pthreads != NULL) + if (pthreads != NULL && pthreads != Py_None) { + if (!PyObject_TypeCheck(pthreads, &pyrf_thread_map__type)) { + PyErr_SetString(PyExc_TypeError, "threads must be a thread_map"); + return NULL; + } threads = ((struct pyrf_thread_map *)pthreads)->threads; + } - if (pcpus != NULL) + if (pcpus != NULL && pcpus != Py_None) { + if (!PyObject_TypeCheck(pcpus, &pyrf_cpu_map__type)) { + PyErr_SetString(PyExc_TypeError, "cpus must be a cpu_map"); + return NULL; + } cpus = ((struct pyrf_cpu_map *)pcpus)->cpus; + } evsel->core.attr.inherit = inherit; /* @@ -1096,21 +1162,26 @@ static PyObject *pyrf_evsel__open(struct pyrf_evsel *pevsel, static PyObject *pyrf_evsel__cpus(struct pyrf_evsel *pevsel) { - struct pyrf_cpu_map *pcpu_map = PyObject_New(struct pyrf_cpu_map, &pyrf_cpu_map__type); + struct pyrf_cpu_map *pcpu_map; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + pcpu_map = PyObject_New(struct pyrf_cpu_map, &pyrf_cpu_map__type); if (pcpu_map) - pcpu_map->cpus = perf_cpu_map__get(pevsel->evsel.core.cpus); + pcpu_map->cpus = perf_cpu_map__get(pevsel->evsel->core.cpus); return (PyObject *)pcpu_map; } static PyObject *pyrf_evsel__threads(struct pyrf_evsel *pevsel) { - struct pyrf_thread_map *pthread_map = - PyObject_New(struct pyrf_thread_map, &pyrf_thread_map__type); + struct pyrf_thread_map *pthread_map; + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + pthread_map = PyObject_New(struct pyrf_thread_map, &pyrf_thread_map__type); if (pthread_map) - pthread_map->threads = perf_thread_map__get(pevsel->evsel.core.threads); + pthread_map->threads = perf_thread_map__get(pevsel->evsel->core.threads); return (PyObject *)pthread_map; } @@ -1144,14 +1215,12 @@ static int evsel__ensure_counts(struct evsel *evsel) static PyObject *pyrf_evsel__read(struct pyrf_evsel *pevsel, PyObject *args, PyObject *kwargs) { - struct evsel *evsel = &pevsel->evsel; + struct evsel *evsel = pevsel->evsel; int cpu = 0, cpu_idx, thread = 0, thread_idx; struct perf_counts_values *old_count, *new_count; - struct pyrf_counts_values *count_values = PyObject_New(struct pyrf_counts_values, - &pyrf_counts_values__type); + struct pyrf_counts_values *count_values; - if (!count_values) - return NULL; + CHECK_INITIALIZED(evsel, "evsel"); if (!PyArg_ParseTuple(args, "ii", &cpu, &thread)) return NULL; @@ -1171,6 +1240,10 @@ static PyObject *pyrf_evsel__read(struct pyrf_evsel *pevsel, if (evsel__ensure_counts(evsel)) return PyErr_NoMemory(); + count_values = PyObject_New(struct pyrf_counts_values, &pyrf_counts_values__type); + if (!count_values) + return NULL; + /* Set up pointers to the old and newly read counter values. */ old_count = perf_counts(evsel->prev_raw_counts, cpu_idx, thread_idx); new_count = perf_counts(evsel->counts, cpu_idx, thread_idx); @@ -1189,7 +1262,10 @@ static PyObject *pyrf_evsel__read(struct pyrf_evsel *pevsel, static PyObject *pyrf_evsel__str(PyObject *self) { struct pyrf_evsel *pevsel = (void *)self; - struct evsel *evsel = &pevsel->evsel; + struct evsel *evsel = pevsel->evsel; + + if (!evsel) + return PyUnicode_FromString("evsel(uninitialized)"); return PyUnicode_FromFormat("evsel(%s/%s/)", evsel__pmu_name(evsel), evsel__name(evsel)); } @@ -1222,30 +1298,292 @@ static PyMethodDef pyrf_evsel__methods[] = { { .ml_name = NULL, } }; -#define evsel_member_def(member, ptype, help) \ - { #member, ptype, \ - offsetof(struct pyrf_evsel, evsel.member), \ - 0, help } +static PyObject *pyrf_evsel__get_tracking(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; -#define evsel_attr_member_def(member, ptype, help) \ - { #member, ptype, \ - offsetof(struct pyrf_evsel, evsel.core.attr.member), \ - 0, help } + CHECK_INITIALIZED(pevsel->evsel, "evsel"); -static PyMemberDef pyrf_evsel__members[] = { - evsel_member_def(tracking, T_BOOL, "tracking event."), - evsel_attr_member_def(type, T_UINT, "attribute type."), - evsel_attr_member_def(size, T_UINT, "attribute size."), - evsel_attr_member_def(config, T_ULONGLONG, "attribute config."), - evsel_attr_member_def(sample_period, T_ULONGLONG, "attribute sample_period."), - evsel_attr_member_def(sample_type, T_ULONGLONG, "attribute sample_type."), - evsel_attr_member_def(read_format, T_ULONGLONG, "attribute read_format."), - evsel_attr_member_def(wakeup_events, T_UINT, "attribute wakeup_events."), - { .name = NULL, }, + if (pevsel->evsel->tracking) + Py_RETURN_TRUE; + else + Py_RETURN_FALSE; +} + +static int pyrf_evsel__set_tracking(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + int is_true; + + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); + + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete attribute"); + return -1; + } + + is_true = PyObject_IsTrue(val); + if (is_true < 0) + return -1; + + pevsel->evsel->tracking = is_true; + return 0; +} + +static int pyrf_evsel__set_attr_config(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + unsigned long long new_val; + + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); + + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete attribute"); + return -1; + } + + new_val = PyLong_AsUnsignedLongLong(val); + if (PyErr_Occurred()) + return -1; + + pevsel->evsel->core.attr.config = new_val; + return 0; +} + +static PyObject *pyrf_evsel__get_attr_config(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + return PyLong_FromUnsignedLongLong(pevsel->evsel->core.attr.config); +} + +static int pyrf_evsel__set_attr_read_format(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + unsigned long long new_val; + + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); + + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete attribute"); + return -1; + } + + new_val = PyLong_AsUnsignedLongLong(val); + if (PyErr_Occurred()) + return -1; + + pevsel->evsel->core.attr.read_format = new_val; + return 0; +} + +static PyObject *pyrf_evsel__get_attr_read_format(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + return PyLong_FromUnsignedLongLong(pevsel->evsel->core.attr.read_format); +} + +static int pyrf_evsel__set_attr_sample_period(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + unsigned long long new_val; + + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); + + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete attribute"); + return -1; + } + + new_val = PyLong_AsUnsignedLongLong(val); + if (PyErr_Occurred()) + return -1; + + pevsel->evsel->core.attr.sample_period = new_val; + return 0; +} + +static PyObject *pyrf_evsel__get_attr_sample_period(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + return PyLong_FromUnsignedLongLong(pevsel->evsel->core.attr.sample_period); +} + +static int pyrf_evsel__set_attr_sample_type(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + unsigned long long new_val; + + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); + + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete attribute"); + return -1; + } + + new_val = PyLong_AsUnsignedLongLong(val); + if (PyErr_Occurred()) + return -1; + + pevsel->evsel->core.attr.sample_type = new_val; + return 0; +} + +static PyObject *pyrf_evsel__get_attr_sample_type(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + return PyLong_FromUnsignedLongLong(pevsel->evsel->core.attr.sample_type); +} + +static PyObject *pyrf_evsel__get_attr_size(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + return PyLong_FromUnsignedLong(pevsel->evsel->core.attr.size); +} + +static int pyrf_evsel__set_attr_type(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + unsigned long new_val; + + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); + + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete attribute"); + return -1; + } + + new_val = PyLong_AsUnsignedLong(val); + if (PyErr_Occurred()) + return -1; + + pevsel->evsel->core.attr.type = new_val; + return 0; +} + +static PyObject *pyrf_evsel__get_attr_type(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + return PyLong_FromUnsignedLong(pevsel->evsel->core.attr.type); +} + +static int pyrf_evsel__set_attr_wakeup_events(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + unsigned long new_val; + + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); + + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete attribute"); + return -1; + } + + new_val = PyLong_AsUnsignedLong(val); + if (PyErr_Occurred()) + return -1; + + pevsel->evsel->core.attr.wakeup_events = new_val; + return 0; +} + +static PyObject *pyrf_evsel__get_attr_wakeup_events(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + return PyLong_FromUnsignedLong(pevsel->evsel->core.attr.wakeup_events); +} + +static PyGetSetDef pyrf_evsel__getset[] = { + { + .name = "tracking", + .get = pyrf_evsel__get_tracking, + .set = pyrf_evsel__set_tracking, + .doc = "tracking event.", + }, + { + .name = "config", + .get = pyrf_evsel__get_attr_config, + .set = pyrf_evsel__set_attr_config, + .doc = "attribute config.", + }, + { + .name = "read_format", + .get = pyrf_evsel__get_attr_read_format, + .set = pyrf_evsel__set_attr_read_format, + .doc = "attribute read_format.", + }, + { + .name = "sample_period", + .get = pyrf_evsel__get_attr_sample_period, + .set = pyrf_evsel__set_attr_sample_period, + .doc = "attribute sample_period.", + }, + { + .name = "sample_type", + .get = pyrf_evsel__get_attr_sample_type, + .set = pyrf_evsel__set_attr_sample_type, + .doc = "attribute sample_type.", + }, + { + .name = "size", + .get = pyrf_evsel__get_attr_size, + .doc = "attribute size.", + }, + { + .name = "type", + .get = pyrf_evsel__get_attr_type, + .set = pyrf_evsel__set_attr_type, + .doc = "attribute type.", + }, + { + .name = "wakeup_events", + .get = pyrf_evsel__get_attr_wakeup_events, + .set = pyrf_evsel__set_attr_wakeup_events, + .doc = "attribute wakeup_events.", + }, + { .name = NULL}, }; static const char pyrf_evsel__doc[] = PyDoc_STR("perf event selector list object."); +static PyObject *pyrf_evsel__getattro(struct pyrf_evsel *pevsel, PyObject *attr_name) +{ + if (!pevsel->evsel) { + PyErr_SetString(PyExc_ValueError, "evsel not initialized"); + return NULL; + } + return PyObject_GenericGetAttr((PyObject *) pevsel, attr_name); +} + +static int pyrf_evsel__setattro(struct pyrf_evsel *pevsel, PyObject *attr_name, PyObject *value) +{ + if (!pevsel->evsel) { + PyErr_SetString(PyExc_ValueError, "evsel not initialized"); + return -1; + } + return PyObject_GenericSetAttr((PyObject *) pevsel, attr_name, value); +} + static PyTypeObject pyrf_evsel__type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "perf.evsel", @@ -1253,23 +1591,35 @@ static PyTypeObject pyrf_evsel__type = { .tp_dealloc = (destructor)pyrf_evsel__delete, .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, .tp_doc = pyrf_evsel__doc, - .tp_members = pyrf_evsel__members, + .tp_getset = pyrf_evsel__getset, .tp_methods = pyrf_evsel__methods, .tp_init = (initproc)pyrf_evsel__init, .tp_str = pyrf_evsel__str, .tp_repr = pyrf_evsel__str, + .tp_getattro = (getattrofunc) pyrf_evsel__getattro, + .tp_setattro = (setattrofunc) pyrf_evsel__setattro, }; +static PyObject *pyrf_evsel__new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + struct pyrf_evsel *pevsel; + + pevsel = (struct pyrf_evsel *)PyType_GenericNew(type, args, kwargs); + if (pevsel) + pevsel->evsel = NULL; + return (PyObject *)pevsel; +} + static int pyrf_evsel__setup_types(void) { - pyrf_evsel__type.tp_new = PyType_GenericNew; + pyrf_evsel__type.tp_new = pyrf_evsel__new; return PyType_Ready(&pyrf_evsel__type); } struct pyrf_evlist { PyObject_HEAD - struct evlist evlist; + struct evlist *evlist; }; static int pyrf_evlist__init(struct pyrf_evlist *pevlist, @@ -1279,40 +1629,55 @@ static int pyrf_evlist__init(struct pyrf_evlist *pevlist, struct perf_cpu_map *cpus; struct perf_thread_map *threads; - if (!PyArg_ParseTuple(args, "OO", &pcpus, &pthreads)) + if (!PyArg_ParseTuple(args, "O!O!", + &pyrf_cpu_map__type, &pcpus, + &pyrf_thread_map__type, &pthreads)) return -1; + evlist__put(pevlist->evlist); + pevlist->evlist = evlist__new(); + if (!pevlist->evlist) { + PyErr_NoMemory(); + return -1; + } threads = ((struct pyrf_thread_map *)pthreads)->threads; cpus = ((struct pyrf_cpu_map *)pcpus)->cpus; - evlist__init(&pevlist->evlist, cpus, threads); + perf_evlist__set_maps(evlist__core(pevlist->evlist), cpus, threads); + return 0; } static void pyrf_evlist__delete(struct pyrf_evlist *pevlist) { - evlist__exit(&pevlist->evlist); + evlist__put(pevlist->evlist); Py_TYPE(pevlist)->tp_free((PyObject*)pevlist); } static PyObject *pyrf_evlist__all_cpus(struct pyrf_evlist *pevlist) { - struct pyrf_cpu_map *pcpu_map = PyObject_New(struct pyrf_cpu_map, &pyrf_cpu_map__type); + struct pyrf_cpu_map *pcpu_map; + + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + pcpu_map = PyObject_New(struct pyrf_cpu_map, &pyrf_cpu_map__type); if (pcpu_map) - pcpu_map->cpus = perf_cpu_map__get(pevlist->evlist.core.all_cpus); + pcpu_map->cpus = perf_cpu_map__get(evlist__core(pevlist->evlist)->all_cpus); return (PyObject *)pcpu_map; } static PyObject *pyrf_evlist__metrics(struct pyrf_evlist *pevlist) { - PyObject *list = PyList_New(/*len=*/0); + PyObject *list; struct rb_node *node; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + list = PyList_New(/*len=*/0); if (!list) return NULL; - for (node = rb_first_cached(&pevlist->evlist.metric_events.entries); node; + for (node = rb_first_cached(&evlist__metric_events(pevlist->evlist)->entries); node; node = rb_next(node)) { struct metric_event *me = container_of(node, struct metric_event, nd); struct list_head *pos; @@ -1415,10 +1780,12 @@ static PyObject *pyrf_evlist__compute_metric(struct pyrf_evlist *pevlist, double result = 0; struct evsel *metric_evsel = NULL; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + if (!PyArg_ParseTuple(args, "sii", &metric, &cpu, &thread)) return NULL; - for (node = rb_first_cached(&pevlist->evlist.metric_events.entries); + for (node = rb_first_cached(&evlist__metric_events(pevlist->evlist)->entries); mexp == NULL && node; node = rb_next(node)) { struct metric_event *me = container_of(node, struct metric_event, nd); @@ -1434,7 +1801,7 @@ static PyObject *pyrf_evlist__compute_metric(struct pyrf_evlist *pevlist, if (e->metric_events[0] == NULL) continue; - evlist__for_each_entry(&pevlist->evlist, pos2) { + evlist__for_each_entry(pevlist->evlist, pos2) { if (pos2->metric_leader != e->metric_events[0]) continue; cpu_idx = perf_cpu_map__idx(pos2->core.cpus, @@ -1479,15 +1846,18 @@ done: static PyObject *pyrf_evlist__mmap(struct pyrf_evlist *pevlist, PyObject *args, PyObject *kwargs) { - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; static char *kwlist[] = { "pages", "overwrite", NULL }; int pages = 128, overwrite = false; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + evlist = pevlist->evlist; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|ii", kwlist, &pages, &overwrite)) return NULL; - if (evlist__mmap(evlist, pages) < 0) { + if (evlist__do_mmap(evlist, pages) < 0) { PyErr_SetFromErrno(PyExc_OSError); return NULL; } @@ -1499,10 +1869,13 @@ static PyObject *pyrf_evlist__mmap(struct pyrf_evlist *pevlist, static PyObject *pyrf_evlist__poll(struct pyrf_evlist *pevlist, PyObject *args, PyObject *kwargs) { - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; static char *kwlist[] = { "timeout", NULL }; int timeout = -1, n; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + evlist = pevlist->evlist; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|i", kwlist, &timeout)) return NULL; @@ -1519,13 +1892,20 @@ static PyObject *pyrf_evlist__get_pollfd(struct pyrf_evlist *pevlist, PyObject *args __maybe_unused, PyObject *kwargs __maybe_unused) { - struct evlist *evlist = &pevlist->evlist; - PyObject *list = PyList_New(0); + struct evlist *evlist; + PyObject *list; int i; - for (i = 0; i < evlist->core.pollfd.nr; ++i) { + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + evlist = pevlist->evlist; + list = PyList_New(0); + if (!list) + return NULL; + + for (i = 0; i < evlist__core(evlist)->pollfd.nr; ++i) { PyObject *file; - file = PyFile_FromFd(evlist->core.pollfd.entries[i].fd, "perf", "r", -1, + file = PyFile_FromFd(evlist__core(evlist)->pollfd.entries[i].fd, "perf", "r", -1, NULL, NULL, NULL, 0); if (file == NULL) goto free_list; @@ -1540,6 +1920,7 @@ static PyObject *pyrf_evlist__get_pollfd(struct pyrf_evlist *pevlist, return list; free_list: + Py_XDECREF(list); return PyErr_NoMemory(); } @@ -1548,27 +1929,33 @@ static PyObject *pyrf_evlist__add(struct pyrf_evlist *pevlist, PyObject *args, PyObject *kwargs __maybe_unused) { - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; PyObject *pevsel; struct evsel *evsel; - if (!PyArg_ParseTuple(args, "O", &pevsel)) + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + evlist = pevlist->evlist; + if (!PyArg_ParseTuple(args, "O!", &pyrf_evsel__type, &pevsel)) return NULL; - Py_INCREF(pevsel); - evsel = &((struct pyrf_evsel *)pevsel)->evsel; - evsel->core.idx = evlist->core.nr_entries; - evlist__add(evlist, evsel); + CHECK_INITIALIZED(((struct pyrf_evsel *)pevsel)->evsel, "evsel"); + + evsel = ((struct pyrf_evsel *)pevsel)->evsel; + CHECK_INITIALIZED(evsel, "evsel"); + + evsel->core.idx = evlist__nr_entries(evlist); + evlist__add(evlist, evsel__get(evsel)); - return Py_BuildValue("i", evlist->core.nr_entries); + return Py_BuildValue("i", evlist__nr_entries(evlist)); } static struct mmap *get_md(struct evlist *evlist, int cpu) { int i; - for (i = 0; i < evlist->core.nr_mmaps; i++) { - struct mmap *md = &evlist->mmap[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + struct mmap *md = &evlist__mmap(evlist)[i]; if (md->core.cpu.cpu == cpu) return md; @@ -1580,13 +1967,16 @@ static struct mmap *get_md(struct evlist *evlist, int cpu) static PyObject *pyrf_evlist__read_on_cpu(struct pyrf_evlist *pevlist, PyObject *args, PyObject *kwargs) { - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; union perf_event *event; int sample_id_all = 1, cpu; static char *kwlist[] = { "cpu", "sample_id_all", NULL }; struct mmap *md; int err; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + evlist = pevlist->evlist; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "i|i", kwlist, &cpu, &sample_id_all)) return NULL; @@ -1616,7 +2006,7 @@ static PyObject *pyrf_evlist__read_on_cpu(struct pyrf_evlist *pevlist, return Py_None; } - pevent->evsel = evsel; + pevent->evsel = evsel__get(evsel); perf_mmap__consume(&md->core); @@ -1637,8 +2027,11 @@ end: static PyObject *pyrf_evlist__open(struct pyrf_evlist *pevlist, PyObject *args, PyObject *kwargs) { - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; + + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + evlist = pevlist->evlist; if (evlist__open(evlist) < 0) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -1650,8 +2043,11 @@ static PyObject *pyrf_evlist__open(struct pyrf_evlist *pevlist, static PyObject *pyrf_evlist__close(struct pyrf_evlist *pevlist) { - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; + + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + evlist = pevlist->evlist; evlist__close(evlist); Py_INCREF(Py_None); @@ -1676,8 +2072,11 @@ static PyObject *pyrf_evlist__config(struct pyrf_evlist *pevlist) .no_buffering = true, .no_inherit = true, }; - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + evlist = pevlist->evlist; evlist__config(evlist, &opts, &callchain_param); Py_INCREF(Py_None); return Py_None; @@ -1685,14 +2084,16 @@ static PyObject *pyrf_evlist__config(struct pyrf_evlist *pevlist) static PyObject *pyrf_evlist__disable(struct pyrf_evlist *pevlist) { - evlist__disable(&pevlist->evlist); + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + evlist__disable(pevlist->evlist); Py_INCREF(Py_None); return Py_None; } static PyObject *pyrf_evlist__enable(struct pyrf_evlist *pevlist) { - evlist__enable(&pevlist->evlist); + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + evlist__enable(pevlist->evlist); Py_INCREF(Py_None); return Py_None; } @@ -1783,7 +2184,21 @@ static Py_ssize_t pyrf_evlist__length(PyObject *obj) { struct pyrf_evlist *pevlist = (void *)obj; - return pevlist->evlist.core.nr_entries; + if (!pevlist->evlist) + return 0; + + return evlist__nr_entries(pevlist->evlist); +} + +static PyObject *pyrf_evsel__from_evsel(struct evsel *evsel) +{ + struct pyrf_evsel *pevsel = PyObject_New(struct pyrf_evsel, &pyrf_evsel__type); + + if (!pevsel) + return NULL; + + pevsel->evsel = evsel__get(evsel); + return (PyObject *)pevsel; } static PyObject *pyrf_evlist__item(PyObject *obj, Py_ssize_t i) @@ -1791,17 +2206,16 @@ static PyObject *pyrf_evlist__item(PyObject *obj, Py_ssize_t i) struct pyrf_evlist *pevlist = (void *)obj; struct evsel *pos; - if (i >= pevlist->evlist.core.nr_entries) { + if (!pevlist->evlist || i >= evlist__nr_entries(pevlist->evlist)) { PyErr_SetString(PyExc_IndexError, "Index out of range"); return NULL; } - evlist__for_each_entry(&pevlist->evlist, pos) { + evlist__for_each_entry(pevlist->evlist, pos) { if (i-- == 0) break; } - - return Py_BuildValue("O", container_of(pos, struct pyrf_evsel, evsel)); + return pyrf_evsel__from_evsel(pos); } static PyObject *pyrf_evlist__str(PyObject *self) @@ -1812,8 +2226,11 @@ static PyObject *pyrf_evlist__str(PyObject *self) bool first = true; PyObject *result; + if (!pevlist->evlist) + return PyUnicode_FromString("evlist(uninitialized)"); + strbuf_addstr(&sb, "evlist(["); - evlist__for_each_entry(&pevlist->evlist, pos) { + evlist__for_each_entry(pevlist->evlist, pos) { if (!first) strbuf_addch(&sb, ','); if (!pos->pmu) @@ -1835,6 +2252,24 @@ static PySequenceMethods pyrf_evlist__sequence_methods = { static const char pyrf_evlist__doc[] = PyDoc_STR("perf event selector list object."); +static PyObject *pyrf_evlist__getattro(struct pyrf_evlist *pevlist, PyObject *attr_name) +{ + if (!pevlist->evlist) { + PyErr_SetString(PyExc_ValueError, "evlist not initialized"); + return NULL; + } + return PyObject_GenericGetAttr((PyObject *) pevlist, attr_name); +} + +static int pyrf_evlist__setattro(struct pyrf_evlist *pevlist, PyObject *attr_name, PyObject *value) +{ + if (!pevlist->evlist) { + PyErr_SetString(PyExc_ValueError, "evlist not initialized"); + return -1; + } + return PyObject_GenericSetAttr((PyObject *) pevlist, attr_name, value); +} + static PyTypeObject pyrf_evlist__type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "perf.evlist", @@ -1847,11 +2282,23 @@ static PyTypeObject pyrf_evlist__type = { .tp_init = (initproc)pyrf_evlist__init, .tp_repr = pyrf_evlist__str, .tp_str = pyrf_evlist__str, + .tp_getattro = (getattrofunc) pyrf_evlist__getattro, + .tp_setattro = (setattrofunc) pyrf_evlist__setattro, }; +static PyObject *pyrf_evlist__new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + struct pyrf_evlist *pevlist; + + pevlist = (struct pyrf_evlist *)PyType_GenericNew(type, args, kwargs); + if (pevlist) + pevlist->evlist = NULL; + return (PyObject *)pevlist; +} + static int pyrf_evlist__setup_types(void) { - pyrf_evlist__type.tp_new = PyType_GenericNew; + pyrf_evlist__type.tp_new = pyrf_evlist__new; return PyType_Ready(&pyrf_evlist__type); } @@ -1954,157 +2401,79 @@ static PyObject *pyrf__tracepoint(struct pyrf_evsel *pevsel, return PyLong_FromLong(tp_pmu__id(sys, name)); } -static PyObject *pyrf_evsel__from_evsel(struct evsel *evsel) -{ - struct pyrf_evsel *pevsel = PyObject_New(struct pyrf_evsel, &pyrf_evsel__type); - - if (!pevsel) - return NULL; - - memset(&pevsel->evsel, 0, sizeof(pevsel->evsel)); - evsel__init(&pevsel->evsel, &evsel->core.attr, evsel->core.idx); - - evsel__clone(&pevsel->evsel, evsel); - if (evsel__is_group_leader(evsel)) - evsel__set_leader(&pevsel->evsel, &pevsel->evsel); - return (PyObject *)pevsel; -} - -static int evlist__pos(struct evlist *evlist, struct evsel *evsel) -{ - struct evsel *pos; - int idx = 0; - - evlist__for_each_entry(evlist, pos) { - if (evsel == pos) - return idx; - idx++; - } - return -1; -} - -static struct evsel *evlist__at(struct evlist *evlist, int idx) -{ - struct evsel *pos; - int idx2 = 0; - - evlist__for_each_entry(evlist, pos) { - if (idx == idx2) - return pos; - idx2++; - } - return NULL; -} - static PyObject *pyrf_evlist__from_evlist(struct evlist *evlist) { struct pyrf_evlist *pevlist = PyObject_New(struct pyrf_evlist, &pyrf_evlist__type); - struct evsel *pos; - struct rb_node *node; if (!pevlist) return NULL; - memset(&pevlist->evlist, 0, sizeof(pevlist->evlist)); - evlist__init(&pevlist->evlist, evlist->core.all_cpus, evlist->core.threads); - evlist__for_each_entry(evlist, pos) { - struct pyrf_evsel *pevsel = (void *)pyrf_evsel__from_evsel(pos); - - evlist__add(&pevlist->evlist, &pevsel->evsel); - } - evlist__for_each_entry(&pevlist->evlist, pos) { - struct evsel *leader = evsel__leader(pos); - - if (pos != leader) { - int idx = evlist__pos(evlist, leader); - - if (idx >= 0) - evsel__set_leader(pos, evlist__at(&pevlist->evlist, idx)); - else if (leader == NULL) - evsel__set_leader(pos, pos); - } - - leader = pos->metric_leader; - - if (pos != leader) { - int idx = evlist__pos(evlist, leader); - - if (idx >= 0) - pos->metric_leader = evlist__at(&pevlist->evlist, idx); - else if (leader == NULL) - pos->metric_leader = pos; - } - } - metricgroup__copy_metric_events(&pevlist->evlist, /*cgrp=*/NULL, - &pevlist->evlist.metric_events, - &evlist->metric_events); - for (node = rb_first_cached(&pevlist->evlist.metric_events.entries); node; - node = rb_next(node)) { - struct metric_event *me = container_of(node, struct metric_event, nd); - struct list_head *mpos; - int idx = evlist__pos(evlist, me->evsel); - - if (idx >= 0) - me->evsel = evlist__at(&pevlist->evlist, idx); - list_for_each(mpos, &me->head) { - struct metric_expr *e = container_of(mpos, struct metric_expr, nd); - - for (int j = 0; e->metric_events[j]; j++) { - idx = evlist__pos(evlist, e->metric_events[j]); - if (idx >= 0) - e->metric_events[j] = evlist__at(&pevlist->evlist, idx); - } - } - } + pevlist->evlist = evlist__get(evlist); return (PyObject *)pevlist; } static PyObject *pyrf__parse_events(PyObject *self, PyObject *args) { const char *input; - struct evlist evlist = {}; + struct evlist *evlist = evlist__new(); struct parse_events_error err; PyObject *result; PyObject *pcpus = NULL, *pthreads = NULL; struct perf_cpu_map *cpus; struct perf_thread_map *threads; - if (!PyArg_ParseTuple(args, "s|OO", &input, &pcpus, &pthreads)) + if (!evlist) + return PyErr_NoMemory(); + + if (!PyArg_ParseTuple(args, "s|OO", &input, &pcpus, &pthreads)) { + evlist__put(evlist); return NULL; + } - threads = pthreads ? ((struct pyrf_thread_map *)pthreads)->threads : NULL; - cpus = pcpus ? ((struct pyrf_cpu_map *)pcpus)->cpus : NULL; + threads = (pthreads && pthreads != Py_None) ? + ((struct pyrf_thread_map *)pthreads)->threads : NULL; + cpus = (pcpus && pcpus != Py_None) ? + ((struct pyrf_cpu_map *)pcpus)->cpus : NULL; parse_events_error__init(&err); - evlist__init(&evlist, cpus, threads); - if (parse_events(&evlist, input, &err)) { + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); + if (parse_events(evlist, input, &err)) { parse_events_error__print(&err, input); PyErr_SetFromErrno(PyExc_OSError); + evlist__put(evlist); return NULL; } - result = pyrf_evlist__from_evlist(&evlist); - evlist__exit(&evlist); + result = pyrf_evlist__from_evlist(evlist); + evlist__put(evlist); return result; } static PyObject *pyrf__parse_metrics(PyObject *self, PyObject *args) { const char *input, *pmu = NULL; - struct evlist evlist = {}; + struct evlist *evlist = evlist__new(); PyObject *result; PyObject *pcpus = NULL, *pthreads = NULL; struct perf_cpu_map *cpus; struct perf_thread_map *threads; int ret; - if (!PyArg_ParseTuple(args, "s|sOO", &input, &pmu, &pcpus, &pthreads)) + if (!evlist) + return PyErr_NoMemory(); + + if (!PyArg_ParseTuple(args, "s|sOO", &input, &pmu, &pcpus, &pthreads)) { + evlist__put(evlist); return NULL; + } - threads = pthreads ? ((struct pyrf_thread_map *)pthreads)->threads : NULL; - cpus = pcpus ? ((struct pyrf_cpu_map *)pcpus)->cpus : NULL; + threads = (pthreads && pthreads != Py_None) ? + ((struct pyrf_thread_map *)pthreads)->threads : NULL; + cpus = (pcpus && pcpus != Py_None) ? + ((struct pyrf_cpu_map *)pcpus)->cpus : NULL; - evlist__init(&evlist, cpus, threads); - ret = metricgroup__parse_groups(&evlist, pmu ?: "all", input, + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); + ret = metricgroup__parse_groups(evlist, pmu ?: "all", + /*cputype_filter=*/false, input, /*metric_no_group=*/ false, /*metric_no_merge=*/ false, /*metric_no_threshold=*/ true, @@ -2112,12 +2481,13 @@ static PyObject *pyrf__parse_metrics(PyObject *self, PyObject *args) /*system_wide=*/true, /*hardware_aware_grouping=*/ false); if (ret) { + evlist__put(evlist); errno = -ret; PyErr_SetFromErrno(PyExc_OSError); return NULL; } - result = pyrf_evlist__from_evlist(&evlist); - evlist__exit(&evlist); + result = pyrf_evlist__from_evlist(evlist); + evlist__put(evlist); return result; } @@ -2161,6 +2531,8 @@ static int pyrf__metrics_cb(const struct pmu_metric *pm, Py_XDECREF(dict); return -ENOMEM; } + Py_DECREF(key); + Py_DECREF(value); if (!add_to_dict(dict, "MetricName", pm->metric_name) || !add_to_dict(dict, "PMU", pm->pmu) || diff --git a/tools/perf/util/record.c b/tools/perf/util/record.c index e867de8ddaaa41..38e8aee3106b7b 100644 --- a/tools/perf/util/record.c +++ b/tools/perf/util/record.c @@ -99,7 +99,7 @@ void evlist__config(struct evlist *evlist, struct record_opts *opts, struct call bool use_comm_exec; bool sample_id = opts->sample_id; - if (perf_cpu_map__cpu(evlist->core.user_requested_cpus, 0).cpu < 0) + if (perf_cpu_map__cpu(evlist__core(evlist)->user_requested_cpus, 0).cpu < 0) opts->no_inherit = true; use_comm_exec = perf_can_comm_exec(); @@ -122,7 +122,7 @@ void evlist__config(struct evlist *evlist, struct record_opts *opts, struct call */ use_sample_identifier = perf_can_sample_identifier(); sample_id = true; - } else if (evlist->core.nr_entries > 1) { + } else if (evlist__nr_entries(evlist) > 1) { struct evsel *first = evlist__first(evlist); evlist__for_each_entry(evlist, evsel) { @@ -237,7 +237,8 @@ bool evlist__can_select_event(struct evlist *evlist, const char *str) evsel = evlist__last(temp_evlist); - if (!evlist || perf_cpu_map__is_any_cpu_or_is_empty(evlist->core.user_requested_cpus)) { + if (!evlist || + perf_cpu_map__is_any_cpu_or_is_empty(evlist__core(evlist)->user_requested_cpus)) { struct perf_cpu_map *cpus = perf_cpu_map__new_online_cpus(); if (cpus) @@ -245,7 +246,7 @@ bool evlist__can_select_event(struct evlist *evlist, const char *str) perf_cpu_map__put(cpus); } else { - cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, 0); + cpu = perf_cpu_map__cpu(evlist__core(evlist)->user_requested_cpus, 0); } while (1) { @@ -264,7 +265,7 @@ bool evlist__can_select_event(struct evlist *evlist, const char *str) ret = true; out_delete: - evlist__delete(temp_evlist); + evlist__put(temp_evlist); return ret; } diff --git a/tools/perf/util/s390-sample-raw.c b/tools/perf/util/s390-sample-raw.c index 52bbca5c56c8e1..01111c4e34880d 100644 --- a/tools/perf/util/s390-sample-raw.c +++ b/tools/perf/util/s390-sample-raw.c @@ -12,25 +12,26 @@ * sample was taken from. */ -#include +#include #include #include -#include -#include +#include #include #include -#include +#include +#include +#include "color.h" #include "debug.h" -#include "session.h" #include "evlist.h" -#include "color.h" #include "hashmap.h" -#include "sample-raw.h" +#include "pmu.h" +#include "pmus.h" #include "s390-cpumcf-kernel.h" -#include "util/pmu.h" -#include "util/sample.h" +#include "sample-raw.h" +#include "sample.h" +#include "session.h" static size_t ctrset_size(struct cf_ctrset_entry *set) { diff --git a/tools/perf/util/sample-raw.c b/tools/perf/util/sample-raw.c index e20b73c0c5bd75..f5ae9f46898345 100644 --- a/tools/perf/util/sample-raw.c +++ b/tools/perf/util/sample-raw.c @@ -18,11 +18,11 @@ void evlist__init_trace_event_sample_raw(struct evlist *evlist, struct perf_env uint16_t e_machine = perf_env__e_machine(env, /*e_flags=*/NULL); if (e_machine == EM_S390) { - evlist->trace_event_sample_raw = evlist__s390_sample_raw; + evlist__set_trace_event_sample_raw(evlist, evlist__s390_sample_raw); } else if (e_machine == EM_X86_64 || e_machine == EM_386) { const char *cpuid = perf_env__cpuid(env); if (cpuid && strstarts(cpuid, "AuthenticAMD") && evlist__has_amd_ibs(evlist)) - evlist->trace_event_sample_raw = evlist__amd_sample_raw; + evlist__set_trace_event_sample_raw(evlist, evlist__amd_sample_raw); } } diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c index f391a822480db0..10d8942f86dadf 100644 --- a/tools/perf/util/session.c +++ b/tools/perf/util/session.c @@ -205,7 +205,7 @@ struct perf_session *__perf_session__new(struct perf_data *data, session->machines.host.env = host_env; } if (session->evlist) - session->evlist->session = session; + evlist__set_session(session->evlist, session); session->machines.host.single_address_space = perf_env__single_address_space(session->machines.host.env); @@ -265,7 +265,7 @@ void perf_session__delete(struct perf_session *session) machines__exit(&session->machines); if (session->data) { if (perf_data__is_read(session->data)) - evlist__delete(session->evlist); + evlist__put(session->evlist); perf_data__close(session->data); } #ifdef HAVE_LIBTRACEEVENT @@ -1549,8 +1549,8 @@ static void dump_event(struct evlist *evlist, union perf_event *event, file_offset, file_path, event->header.size, event->header.type); trace_event(event); - if (event->header.type == PERF_RECORD_SAMPLE && evlist->trace_event_sample_raw) - evlist->trace_event_sample_raw(evlist, event, sample); + if (event->header.type == PERF_RECORD_SAMPLE && evlist__trace_event_sample_raw(evlist)) + evlist__trace_event_sample_raw(evlist)(evlist, event, sample); if (sample) evlist__print_tstamp(evlist, event, sample); @@ -1751,7 +1751,7 @@ static int deliver_sample_value(struct evlist *evlist, } if (!storage || sid->evsel == NULL) { - ++evlist->stats.nr_unknown_id; + ++evlist__stats(evlist)->nr_unknown_id; return 0; } @@ -1845,12 +1845,15 @@ static int evlist__deliver_deferred_callchain(struct evlist *evlist, struct evsel *saved_evsel = sample->evsel; sample->evsel = evlist__id2evsel(evlist, sample->id); + if (sample->evsel) + sample->evsel = evsel__get(sample->evsel); ret = tool->callchain_deferred(tool, event, sample, machine); + evsel__put(sample->evsel); sample->evsel = saved_evsel; return ret; } - list_for_each_entry_safe(de, tmp, &evlist->deferred_samples, list) { + list_for_each_entry_safe(de, tmp, evlist__deferred_samples(evlist), list) { struct perf_sample orig_sample; perf_sample__init(&orig_sample, /*all=*/false); @@ -1899,7 +1902,7 @@ static int session__flush_deferred_samples(struct perf_session *session, struct deferred_event *de, *tmp; int ret = 0; - list_for_each_entry_safe(de, tmp, &evlist->deferred_samples, list) { + list_for_each_entry_safe(de, tmp, evlist__deferred_samples(evlist), list) { struct perf_sample sample; perf_sample__init(&sample, /*all=*/false); @@ -1961,17 +1964,16 @@ static int machines__deliver_event(struct machines *machines, sample->evsel = evlist__id2evsel(evlist, sample->id); else assert(sample->evsel == evlist__id2evsel(evlist, sample->id)); - machine = machines__find_for_cpumode(machines, event, sample); switch (event->header.type) { case PERF_RECORD_SAMPLE: if (sample->evsel == NULL) { - ++evlist->stats.nr_unknown_id; + ++evlist__stats(evlist)->nr_unknown_id; return 0; } if (machine == NULL) { - ++evlist->stats.nr_unprocessable_samples; + ++evlist__stats(evlist)->nr_unprocessable_samples; dump_sample(machine, event, sample); return 0; } @@ -1990,7 +1992,7 @@ static int machines__deliver_event(struct machines *machines, } memcpy(de->event, event, sz); de->file_offset = sample->file_offset; - list_add_tail(&de->list, &evlist->deferred_samples); + list_add_tail(&de->list, evlist__deferred_samples(evlist)); return 0; } return evlist__deliver_sample(evlist, tool, event, sample, machine); @@ -2002,7 +2004,7 @@ static int machines__deliver_event(struct machines *machines, return tool->mmap(tool, event, sample, machine); case PERF_RECORD_MMAP2: if (event->header.misc & PERF_RECORD_MISC_PROC_MAP_PARSE_TIMEOUT) - ++evlist->stats.nr_proc_map_timeout; + ++evlist__stats(evlist)->nr_proc_map_timeout; if (!perf_event__check_nul(event->mmap2.filename, (void *)event + event->header.size, "MMAP2", file_offset)) @@ -2047,13 +2049,13 @@ static int machines__deliver_event(struct machines *machines, return tool->exit(tool, event, sample, machine); case PERF_RECORD_LOST: if (tool->lost == perf_event__process_lost) - evlist->stats.total_lost += event->lost.lost; + evlist__stats(evlist)->total_lost += event->lost.lost; return tool->lost(tool, event, sample, machine); case PERF_RECORD_LOST_SAMPLES: if (event->header.misc & PERF_RECORD_MISC_LOST_SAMPLES_BPF) - evlist->stats.total_dropped_samples += event->lost_samples.lost; + evlist__stats(evlist)->total_dropped_samples += event->lost_samples.lost; else if (tool->lost_samples == perf_event__process_lost_samples) - evlist->stats.total_lost_samples += event->lost_samples.lost; + evlist__stats(evlist)->total_lost_samples += event->lost_samples.lost; return tool->lost_samples(tool, event, sample, machine); case PERF_RECORD_READ: dump_read(sample->evsel, event); @@ -2065,11 +2067,11 @@ static int machines__deliver_event(struct machines *machines, case PERF_RECORD_AUX: if (tool->aux == perf_event__process_aux) { if (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) - evlist->stats.total_aux_lost += 1; + evlist__stats(evlist)->total_aux_lost += 1; if (event->aux.flags & PERF_AUX_FLAG_PARTIAL) - evlist->stats.total_aux_partial += 1; + evlist__stats(evlist)->total_aux_partial += 1; if (event->aux.flags & PERF_AUX_FLAG_COLLISION) - evlist->stats.total_aux_collision += 1; + evlist__stats(evlist)->total_aux_collision += 1; } return tool->aux(tool, event, sample, machine); case PERF_RECORD_ITRACE_START: @@ -2105,7 +2107,7 @@ static int machines__deliver_event(struct machines *machines, return evlist__deliver_deferred_callchain(evlist, tool, event, sample, machine); default: - ++evlist->stats.nr_unknown_events; + ++evlist__stats(evlist)->nr_unknown_events; return -1; } } @@ -2517,7 +2519,7 @@ int perf_session__deliver_synth_event(struct perf_session *session, struct evlist *evlist = session->evlist; const struct perf_tool *tool = session->tool; - events_stats__inc(&evlist->stats, event->header.type); + events_stats__inc(evlist__stats(evlist), event->header.type); if (event->header.type >= PERF_RECORD_USER_TYPE_START) return perf_session__process_user_event(session, event, 0, NULL); @@ -2927,7 +2929,7 @@ static s64 perf_session__process_event(struct perf_session *session, return 0; } - events_stats__inc(&evlist->stats, event->header.type); + events_stats__inc(evlist__stats(evlist), event->header.type); if (event->header.type >= PERF_RECORD_USER_TYPE_START) return perf_session__process_user_event(session, event, file_offset, file_path); @@ -2988,7 +2990,7 @@ perf_session__warn_order(const struct perf_session *session) static void perf_session__warn_about_errors(const struct perf_session *session) { - const struct events_stats *stats = &session->evlist->stats; + const struct events_stats *stats = evlist__stats(session->evlist); if (session->tool->lost == perf_event__process_lost && stats->nr_events[PERF_RECORD_LOST] != 0) { @@ -3821,7 +3823,7 @@ size_t perf_session__fprintf_nr_events(struct perf_session *session, FILE *fp) ret = fprintf(fp, "\nAggregated stats:%s\n", msg); - ret += events_stats__fprintf(&session->evlist->stats, fp); + ret += events_stats__fprintf(evlist__stats(session->evlist), fp); return ret; } diff --git a/tools/perf/util/sideband_evlist.c b/tools/perf/util/sideband_evlist.c index 388846f17bc13f..ba043db6cedcb3 100644 --- a/tools/perf/util/sideband_evlist.c +++ b/tools/perf/util/sideband_evlist.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -22,7 +23,7 @@ int evlist__add_sb_event(struct evlist *evlist, struct perf_event_attr *attr, attr->sample_id_all = 1; } - evsel = evsel__new_idx(attr, evlist->core.nr_entries); + evsel = evsel__new_idx(attr, evlist__nr_entries(evlist)); if (!evsel) return -1; @@ -49,14 +50,27 @@ static void *perf_evlist__poll_thread(void *arg) while (!done) { bool got_data = false; - if (evlist->thread.done) + if (evlist__sb_thread_done(evlist)) draining = true; if (!draining) evlist__poll(evlist, 1000); - for (i = 0; i < evlist->core.nr_mmaps; i++) { - struct mmap *map = &evlist->mmap[i]; + /* + * When a thread of the monitored target exits, its per-cpu + * ring-buffer fd is closed and starts returning POLLHUP. Such + * dead fds are never requested for POLLIN, but poll() reports + * POLLHUP/POLLERR unconditionally, so leaving them in the + * pollfd array makes the following evlist__poll() return + * immediately forever, spinning this thread at 100% CPU. + * + * Filter them out here, mirroring what the 'perf record' main + * loop does after fdarray__poll(). + */ + evlist__filter_pollfd(evlist, POLLERR | POLLHUP); + + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + struct mmap *map = &evlist__mmap(evlist)[i]; union perf_event *event; if (perf_mmap__read_init(&map->core)) @@ -102,9 +116,9 @@ int evlist__start_sb_thread(struct evlist *evlist, struct target *target) return 0; if (evlist__create_maps(evlist, target)) - goto out_delete_evlist; + goto out_put_evlist; - if (evlist->core.nr_entries > 1) { + if (evlist__nr_entries(evlist) > 1) { bool can_sample_identifier = perf_can_sample_identifier(); evlist__for_each_entry(evlist, counter) @@ -114,27 +128,27 @@ int evlist__start_sb_thread(struct evlist *evlist, struct target *target) } evlist__for_each_entry(evlist, counter) { - if (evsel__open(counter, evlist->core.user_requested_cpus, - evlist->core.threads) < 0) - goto out_delete_evlist; + if (evsel__open(counter, evlist__core(evlist)->user_requested_cpus, + evlist__core(evlist)->threads) < 0) + goto out_put_evlist; } - if (evlist__mmap(evlist, UINT_MAX)) - goto out_delete_evlist; + if (evlist__do_mmap(evlist, UINT_MAX)) + goto out_put_evlist; evlist__for_each_entry(evlist, counter) { if (evsel__enable(counter)) - goto out_delete_evlist; + goto out_put_evlist; } - evlist->thread.done = 0; - if (pthread_create(&evlist->thread.th, NULL, perf_evlist__poll_thread, evlist)) - goto out_delete_evlist; + evlist__set_sb_thread_done(evlist, 0); + if (pthread_create(evlist__sb_thread_th(evlist), NULL, perf_evlist__poll_thread, evlist)) + goto out_put_evlist; return 0; -out_delete_evlist: - evlist__delete(evlist); +out_put_evlist: + evlist__put(evlist); evlist = NULL; return -1; } @@ -143,7 +157,7 @@ void evlist__stop_sb_thread(struct evlist *evlist) { if (!evlist) return; - evlist->thread.done = 1; - pthread_join(evlist->thread.th, NULL); - evlist__delete(evlist); + evlist__set_sb_thread_done(evlist, 1); + pthread_join(*evlist__sb_thread_th(evlist), NULL); + evlist__put(evlist); } diff --git a/tools/perf/util/sort.c b/tools/perf/util/sort.c index 005e7d85dc4a59..58638ec9ae0ede 100644 --- a/tools/perf/util/sort.c +++ b/tools/perf/util/sort.c @@ -3105,7 +3105,7 @@ static int __sort_dimension__add_hpp_sort(struct sort_dimension *sd, struct hpp_sort_entry *hse = __sort_dimension__alloc_hpp(sd, level); if (hse == NULL) - return -1; + return -ENOMEM; perf_hpp_list__register_sort_field(list, &hse->hpp); return 0; @@ -3118,7 +3118,7 @@ static int __sort_dimension__add_hpp_output(struct sort_dimension *sd, struct hpp_sort_entry *hse = __sort_dimension__alloc_hpp(sd, level); if (hse == NULL) - return -1; + return -ENOMEM; perf_hpp_list__column_register(list, &hse->hpp); return 0; @@ -3487,7 +3487,7 @@ static struct evsel *find_evsel(struct evlist *evlist, char *event_name) if (event_name[0] == '%') { int nr = strtol(event_name+1, NULL, 0); - if (nr > evlist->core.nr_entries) + if (nr > evlist__nr_entries(evlist)) return NULL; evsel = evlist__first(evlist); @@ -3742,14 +3742,18 @@ static int __sort_dimension__add(struct sort_dimension *sd, struct perf_hpp_list *list, int level) { + int ret; + if (sd->taken) return 0; - if (__sort_dimension__add_hpp_sort(sd, list, level) < 0) - return -1; + ret = __sort_dimension__add_hpp_sort(sd, list, level); + if (ret < 0) + return ret; - if (__sort_dimension__update(sd, list) < 0) - return -1; + ret = __sort_dimension__update(sd, list); + if (ret < 0) + return ret; sd->taken = 1; @@ -3767,7 +3771,7 @@ static int __hpp_dimension__add(struct hpp_dimension *hd, fmt = __hpp_dimension__alloc_hpp(hd, level); if (!fmt) - return -1; + return -ENOMEM; hd->taken = 1; hd->was_taken = 1; @@ -3779,14 +3783,18 @@ static int __sort_dimension__add_output(struct perf_hpp_list *list, struct sort_dimension *sd, int level) { + int ret; + if (sd->taken) return 0; - if (__sort_dimension__add_hpp_output(sd, list, level) < 0) - return -1; + ret = __sort_dimension__add_hpp_output(sd, list, level); + if (ret < 0) + return ret; - if (__sort_dimension__update(sd, list) < 0) - return -1; + ret = __sort_dimension__update(sd, list); + if (ret < 0) + return ret; sd->taken = 1; return 0; @@ -3803,7 +3811,7 @@ static int __hpp_dimension__add_output(struct perf_hpp_list *list, fmt = __hpp_dimension__alloc_hpp(hd, level); if (!fmt) - return -1; + return -ENOMEM; hd->taken = 1; perf_hpp_list__column_register(list, fmt); @@ -3869,8 +3877,7 @@ int sort_dimension__add(struct perf_hpp_list *list, const char *tok, strlen(tok))) return -EINVAL; - __sort_dimension__add(sd, list, level); - return 0; + return __sort_dimension__add(sd, list, level); } for (i = 0; i < ARRAY_SIZE(memory_sort_dimensions); i++) { @@ -3882,8 +3889,7 @@ int sort_dimension__add(struct perf_hpp_list *list, const char *tok, if (sort__mode != SORT_MODE__MEMORY) return -EINVAL; - __sort_dimension__add(sd, list, level); - return 0; + return __sort_dimension__add(sd, list, level); } for (i = 0; i < ARRAY_SIZE(hpp_sort_dimensions); i++) { @@ -3973,15 +3979,25 @@ static int setup_sort_list(struct perf_hpp_list *list, char *str, } ret = sort_dimension__add(list, tok, evlist, env, level); - if (ret == -EINVAL) { + switch (ret) { + case 0: + break; + case -EINVAL: if (!cacheline_size() && !strncasecmp(tok, "dcacheline", strlen(tok))) ui__error("The \"dcacheline\" --sort key needs to know the cacheline size and it couldn't be determined on this system"); else ui__error("Invalid --sort key: `%s'", tok); - break; - } else if (ret == -ESRCH) { + goto out; + case -ESRCH: ui__error("Unknown --sort key: `%s'", tok); - break; + goto out; + default: { + char buf[STRERR_BUFSIZE]; + + ui__error("%s for --sort key: `%s'", + str_error_r(-ret, buf, sizeof(buf)), tok); + goto out; + } } prev_level = level; } @@ -3989,6 +4005,7 @@ static int setup_sort_list(struct perf_hpp_list *list, char *str, level = next_level; } while (tmp); +out: return ret; } @@ -4315,15 +4332,26 @@ static int setup_output_list(struct perf_hpp_list *list, char *str) for (tok = strtok_r(str, ", ", &tmp); tok; tok = strtok_r(NULL, ", ", &tmp)) { ret = output_field_add(list, tok, &level); - if (ret == -EINVAL) { - ui__error("Invalid --fields key: `%s'", tok); + switch (ret) { + case 0: break; - } else if (ret == -ESRCH) { + case -EINVAL: + ui__error("Invalid --fields key: `%s'", tok); + goto out; + case -ESRCH: ui__error("Unknown --fields key: `%s'", tok); - break; + goto out; + default: { + char buf[STRERR_BUFSIZE]; + + ui__error("%s for --fields key: `%s'", + str_error_r(-ret, buf, sizeof(buf)), tok); + goto out; + } } } +out: return ret; } diff --git a/tools/perf/util/stat-display.c b/tools/perf/util/stat-display.c index 0a5750bb59fa3b..f94f1324d24ac2 100644 --- a/tools/perf/util/stat-display.c +++ b/tools/perf/util/stat-display.c @@ -667,7 +667,7 @@ static void print_metric_header(struct perf_stat_config *config, /* In case of iostat, print metric header for first root port only */ if (config->iostat_run && - os->evsel->priv != os->evsel->evlist->selected->priv) + os->evsel->priv != evlist__selected(os->evsel->evlist)->priv) return; if (os->evsel->cgrp != os->cgrp) @@ -1126,7 +1126,7 @@ static void print_no_aggr_metric(struct perf_stat_config *config, unsigned int all_idx; struct perf_cpu cpu; - perf_cpu_map__for_each_cpu(cpu, all_idx, evlist->core.user_requested_cpus) { + perf_cpu_map__for_each_cpu(cpu, all_idx, evlist__core(evlist)->user_requested_cpus) { struct evsel *counter; bool first = true; @@ -1543,7 +1543,7 @@ void evlist__print_counters(struct evlist *evlist, struct perf_stat_config *conf evlist__uniquify_evsel_names(evlist, config); if (config->iostat_run) - evlist->selected = evlist__first(evlist); + evlist__set_selected(evlist, evlist__first(evlist)); if (config->interval) prepare_timestamp(config, &os, ts); diff --git a/tools/perf/util/stat-shadow.c b/tools/perf/util/stat-shadow.c index c17373bb0e1e7a..525a3fe4a46ec7 100644 --- a/tools/perf/util/stat-shadow.c +++ b/tools/perf/util/stat-shadow.c @@ -2,20 +2,24 @@ #include #include #include -#include "evsel.h" -#include "stat.h" + +#include + +#include "cgroup.h" #include "color.h" #include "debug.h" -#include "pmu.h" -#include "rblist.h" #include "evlist.h" +#include "evsel.h" #include "expr.h" -#include "metricgroup.h" -#include "cgroup.h" -#include "units.h" +#include "hashmap.h" #include "iostat.h" -#include "util/hashmap.h" +#include "metricgroup.h" +#include "pmu.h" +#include "pmus.h" +#include "rblist.h" +#include "stat.h" #include "tool_pmu.h" +#include "units.h" static bool tool_pmu__is_time_event(const struct perf_stat_config *config, const struct evsel *evsel, int *tool_aggr_idx) @@ -283,7 +287,7 @@ void *perf_stat__print_shadow_stats_metricgroup(struct perf_stat_config *config, void *ctxp = out->ctx; bool header_printed = false; const char *name = NULL; - struct rblist *metric_events = &evsel->evlist->metric_events; + struct rblist *metric_events = evlist__metric_events(evsel->evlist); me = metricgroup__lookup(metric_events, evsel, false); if (me == NULL) @@ -351,5 +355,5 @@ bool perf_stat__skip_metric_event(struct evsel *evsel) if (!evsel->default_metricgroup) return false; - return !metricgroup__lookup(&evsel->evlist->metric_events, evsel, false); + return !metricgroup__lookup(evlist__metric_events(evsel->evlist), evsel, false); } diff --git a/tools/perf/util/stat.c b/tools/perf/util/stat.c index 14d169e22e8f56..25f31a17436828 100644 --- a/tools/perf/util/stat.c +++ b/tools/perf/util/stat.c @@ -1,21 +1,25 @@ // SPDX-License-Identifier: GPL-2.0 +#include "stat.h" + #include -#include #include #include #include + +#include +#include + #include "counts.h" #include "cpumap.h" #include "debug.h" +#include "evlist.h" +#include "evsel.h" +#include "hashmap.h" #include "header.h" -#include "stat.h" +#include "pmu.h" #include "session.h" #include "target.h" -#include "evlist.h" -#include "evsel.h" #include "thread_map.h" -#include "util/hashmap.h" -#include void update_stats(struct stats *stats, u64 val) { @@ -543,8 +547,8 @@ static void evsel__merge_aliases(struct evsel *evsel) struct evlist *evlist = evsel->evlist; struct evsel *alias; - alias = list_prepare_entry(evsel, &(evlist->core.entries), core.node); - list_for_each_entry_continue(alias, &evlist->core.entries, core.node) { + alias = list_prepare_entry(evsel, &(evlist__core(evlist)->entries), core.node); + list_for_each_entry_continue(alias, &evlist__core(evlist)->entries, core.node) { if (alias->first_wildcard_match == evsel) { /* Merge the same events on different PMUs. */ evsel__merge_aggr_counters(evsel, alias); diff --git a/tools/perf/util/stream.c b/tools/perf/util/stream.c index 3de4a613085390..7bccd237834440 100644 --- a/tools/perf/util/stream.c +++ b/tools/perf/util/stream.c @@ -131,7 +131,7 @@ static int evlist__init_callchain_streams(struct evlist *evlist, struct evsel *pos; int i = 0; - BUG_ON(els->nr_evsel < evlist->core.nr_entries); + BUG_ON(els->nr_evsel < evlist__nr_entries(evlist)); evlist__for_each_entry(evlist, pos) { struct hists *hists = evsel__hists(pos); @@ -148,7 +148,7 @@ static int evlist__init_callchain_streams(struct evlist *evlist, struct evlist_streams *evlist__create_streams(struct evlist *evlist, int nr_streams_max) { - int nr_evsel = evlist->core.nr_entries, ret = -1; + int nr_evsel = evlist__nr_entries(evlist), ret = -1; struct evlist_streams *els = evlist_streams__new(nr_evsel, nr_streams_max); diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c index cd379ced19e5b0..35104a56d8e385 100644 --- a/tools/perf/util/symbol.c +++ b/tools/perf/util/symbol.c @@ -2452,8 +2452,7 @@ static bool symbol__read_kptr_restrict(void) { bool value = false; FILE *fp = fopen("/proc/sys/kernel/kptr_restrict", "r"); - bool used_root; - bool cap_syslog = perf_cap__capable(CAP_SYSLOG, &used_root); + bool cap_syslog = perf_cap__capable(CAP_SYSLOG); if (fp != NULL) { char line[8]; diff --git a/tools/perf/util/synthetic-events.c b/tools/perf/util/synthetic-events.c index 5307d707711d87..19c1f3eaf58d8e 100644 --- a/tools/perf/util/synthetic-events.c +++ b/tools/perf/util/synthetic-events.c @@ -1058,7 +1058,7 @@ int perf_event__synthesize_threads(const struct perf_tool *tool, else thread_nr = nr_threads_synthesize; - if (thread_nr <= 1) { + if (thread_nr <= 1 || n <= 1) { err = __perf_event__synthesize_threads(tool, process, machine, needs_mmap, mmap_data, @@ -1104,8 +1104,8 @@ int perf_event__synthesize_threads(const struct perf_tool *tool, } err = 0; out_join: - for (i = 0; i < thread_nr; i++) - pthread_join(synthesize_threads[i], NULL); + for (j = 0; j < i; j++) + pthread_join(synthesize_threads[j], NULL); free(args); free_threads: free(synthesize_threads); @@ -2247,7 +2247,7 @@ int perf_event__synthesize_tracing_data(const struct perf_tool *tool, int fd, st * - write the tracing data from the temp file * to the pipe */ - tdata = tracing_data_get(&evlist->core.entries, fd, true); + tdata = tracing_data_get(&evlist__core(evlist)->entries, fd, true); if (!tdata) return -1; @@ -2404,13 +2404,16 @@ int perf_event__synthesize_stat_events(struct perf_stat_config *config, const st } err = perf_event__synthesize_extra_attr(tool, evlist, process, attrs); - err = perf_event__synthesize_thread_map2(tool, evlist->core.threads, process, NULL); + err = perf_event__synthesize_thread_map2(tool, evlist__core(evlist)->threads, + process, /*machine=*/NULL); if (err < 0) { pr_err("Couldn't synthesize thread map.\n"); return err; } - err = perf_event__synthesize_cpu_map(tool, evlist->core.user_requested_cpus, process, NULL); + err = perf_event__synthesize_cpu_map(tool, + evlist__core(evlist)->user_requested_cpus, + process, /*machine=*/NULL); if (err < 0) { pr_err("Couldn't synthesize thread map.\n"); return err; @@ -2518,7 +2521,7 @@ int perf_event__synthesize_for_pipe(const struct perf_tool *tool, ret += err; #ifdef HAVE_LIBTRACEEVENT - if (have_tracepoints(&evlist->core.entries)) { + if (have_tracepoints(&evlist__core(evlist)->entries)) { int fd = perf_data__fd(data); /* diff --git a/tools/perf/util/thread-stack.c b/tools/perf/util/thread-stack.c index c5ce741b074464..1360f44421ef8b 100644 --- a/tools/perf/util/thread-stack.c +++ b/tools/perf/util/thread-stack.c @@ -642,7 +642,7 @@ void thread_stack__br_sample(struct thread *thread, int cpu, sz -= nr; be = &dst->entries[nr]; nr = min(ts->br_stack_pos, sz); - memcpy(be, &src->entries[0], bsz * ts->br_stack_pos); + memcpy(be, &src->entries[0], bsz * nr); } } diff --git a/tools/perf/util/time-utils.c b/tools/perf/util/time-utils.c index d43c4577d7ebc9..5558a5a0fea4a6 100644 --- a/tools/perf/util/time-utils.c +++ b/tools/perf/util/time-utils.c @@ -473,8 +473,8 @@ int perf_time__parse_for_ranges_reltime(const char *time_str, return -ENOMEM; if (has_percent || reltime) { - if (session->evlist->first_sample_time == 0 && - session->evlist->last_sample_time == 0) { + if (evlist__first_sample_time(session->evlist) == 0 && + evlist__last_sample_time(session->evlist) == 0) { pr_err("HINT: no first/last sample time found in perf data.\n" "Please use latest perf binary to execute 'perf record'\n" "(if '--buildid-all' is enabled, please set '--timestamp-boundary').\n"); @@ -486,8 +486,8 @@ int perf_time__parse_for_ranges_reltime(const char *time_str, num = perf_time__percent_parse_str( ptime_range, size, time_str, - session->evlist->first_sample_time, - session->evlist->last_sample_time); + evlist__first_sample_time(session->evlist), + evlist__last_sample_time(session->evlist)); } else { num = perf_time__parse_strs(ptime_range, time_str, size); } @@ -499,8 +499,8 @@ int perf_time__parse_for_ranges_reltime(const char *time_str, int i; for (i = 0; i < num; i++) { - ptime_range[i].start += session->evlist->first_sample_time; - ptime_range[i].end += session->evlist->first_sample_time; + ptime_range[i].start += evlist__first_sample_time(session->evlist); + ptime_range[i].end += evlist__first_sample_time(session->evlist); } } diff --git a/tools/perf/util/top.c b/tools/perf/util/top.c index b06e10a116bb32..851a26be693153 100644 --- a/tools/perf/util/top.c +++ b/tools/perf/util/top.c @@ -71,7 +71,7 @@ size_t perf_top__header_snprintf(struct perf_top *top, char *bf, size_t size) esamples_percent); } - if (top->evlist->core.nr_entries == 1) { + if (evlist__nr_entries(top->evlist) == 1) { struct evsel *first = evlist__first(top->evlist); ret += SNPRINTF(bf + ret, size - ret, "%" PRIu64 "%s ", (uint64_t)first->core.attr.sample_period, @@ -94,7 +94,7 @@ size_t perf_top__header_snprintf(struct perf_top *top, char *bf, size_t size) else ret += SNPRINTF(bf + ret, size - ret, " (all"); - nr_cpus = perf_cpu_map__nr(top->evlist->core.user_requested_cpus); + nr_cpus = perf_cpu_map__nr(evlist__core(top->evlist)->user_requested_cpus); if (target->cpu_list) ret += SNPRINTF(bf + ret, size - ret, ", CPU%s: %s)", nr_cpus > 1 ? "s" : "", diff --git a/tools/perf/util/trace-event-read.c b/tools/perf/util/trace-event-read.c index ecbbb93f018539..52ed496d92c3bf 100644 --- a/tools/perf/util/trace-event-read.c +++ b/tools/perf/util/trace-event-read.c @@ -25,18 +25,18 @@ static int input_fd; static ssize_t trace_data_size; static bool repipe; -static int __do_read(int fd, void *buf, int size) +static ssize_t __do_read(int fd, void *buf, size_t size) { - int rsize = size; + size_t rsize = size; while (size) { - int ret = read(fd, buf, size); + ssize_t ret = read(fd, buf, size); if (ret <= 0) return -1; if (repipe) { - int retw = write(STDOUT_FILENO, buf, ret); + ssize_t retw = write(STDOUT_FILENO, buf, ret); if (retw <= 0 || retw != ret) { pr_debug("repiping input file"); @@ -51,13 +51,13 @@ static int __do_read(int fd, void *buf, int size) return rsize; } -static int do_read(void *data, int size) +static ssize_t do_read(void *data, size_t size) { - int r; + ssize_t r; r = __do_read(input_fd, data, size); if (r <= 0) { - pr_debug("reading input file (size expected=%d received=%d)", + pr_debug("reading input file (size expected=%zu received=%zd)", size, r); return -1; } @@ -68,10 +68,10 @@ static int do_read(void *data, int size) } /* If it fails, the next read will report it */ -static void skip(int size) +static void skip(size_t size) { char buf[BUFSIZ]; - int r; + size_t r; while (size) { r = size > BUFSIZ ? BUFSIZ : size; @@ -127,6 +127,11 @@ static char *read_string(void) } } + if (size >= (int)sizeof(buf) - 1) { + pr_debug("string too long (max %zu bytes)", sizeof(buf) - 1); + goto out; + } + buf[size++] = c; if (!c) @@ -197,7 +202,7 @@ static int read_header_files(struct tep_handle *pevent) unsigned long long size; char *header_page; char buf[BUFSIZ]; - int ret = 0; + ssize_t ret = 0; if (do_read(buf, 12) < 0) return -1; @@ -245,7 +250,7 @@ static int read_header_files(struct tep_handle *pevent) static int read_ftrace_file(struct tep_handle *pevent, unsigned long long size) { - int ret; + ssize_t ret; char *buf; buf = malloc(size); @@ -271,7 +276,7 @@ out: static int read_event_file(struct tep_handle *pevent, char *sys, unsigned long long size) { - int ret; + ssize_t ret; char *buf; buf = malloc(size); @@ -317,7 +322,7 @@ static int read_event_files(struct tep_handle *pevent) int systems; int count; int i,x; - int ret; + ssize_t ret; systems = read4(pevent); @@ -345,7 +350,7 @@ static int read_saved_cmdline(struct tep_handle *pevent) { unsigned long long size; char *buf; - int ret; + ssize_t ret; /* it can have 0 size */ size = read8(pevent); diff --git a/tools/perf/util/unwind-libdw.c b/tools/perf/util/unwind-libdw.c index 7f35042be56779..63a5c2253174f4 100644 --- a/tools/perf/util/unwind-libdw.c +++ b/tools/perf/util/unwind-libdw.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include #include @@ -173,14 +174,30 @@ static int entry(u64 ip, struct unwind_info *ui) return 0; } -static pid_t next_thread(Dwfl *dwfl, void *arg, void **thread_argp) +static pid_t next_thread(Dwfl *dwfl __maybe_unused, void *arg, void **thread_argp) { + struct dwfl_ui_thread_info *dwfl_ui_ti = arg; + /* We want only single thread to be processed. */ if (*thread_argp != NULL) return 0; + assert(dwfl_ui_ti->ui != NULL); *thread_argp = arg; - return dwfl_pid(dwfl); + return thread__tid(dwfl_ui_ti->ui->thread); +} + +static bool get_thread(Dwfl *dwfl __maybe_unused, pid_t tid, void *arg, + void **thread_argp) +{ + struct dwfl_ui_thread_info *dwfl_ui_ti = arg; + + assert(dwfl_ui_ti->ui != NULL); + if (tid != thread__tid(dwfl_ui_ti->ui->thread)) + return false; + + *thread_argp = arg; + return true; } static int access_dso_mem(struct unwind_info *ui, Dwarf_Addr addr, @@ -306,6 +323,7 @@ static bool libdw_set_initial_registers(Dwfl_Thread *thread, void *arg) static const Dwfl_Thread_Callbacks callbacks = { .next_thread = next_thread, + .get_thread = get_thread, .memory_read = memory_read, .set_initial_registers = libdw_set_initial_registers, }; @@ -400,7 +418,7 @@ int libdw__get_entries(unwind_entry_cb_t cb, void *arg, if (err) goto out; - dwfl_attach_state(dwfl, /*elf=*/NULL, thread__tid(thread), &callbacks, + dwfl_attach_state(dwfl, /*elf=*/NULL, thread__pid(thread), &callbacks, /* Dwfl thread function argument*/dwfl_ui_ti); // Ignore thread already attached error. diff --git a/tools/perf/util/util.c b/tools/perf/util/util.c index 2c2a5c449ffd04..8f7cd32f524dc1 100644 --- a/tools/perf/util/util.c +++ b/tools/perf/util/util.c @@ -378,15 +378,9 @@ int perf_event_paranoid(void) bool perf_event_paranoid_check(int max_level) { - bool used_root; - - if (perf_cap__capable(CAP_SYS_ADMIN, &used_root)) - return true; - - if (!used_root && perf_cap__capable(CAP_PERFMON, &used_root)) - return true; - - return perf_event_paranoid() <= max_level; + return perf_cap__capable(CAP_SYS_ADMIN) || + perf_cap__capable(CAP_PERFMON) || + perf_event_paranoid() <= max_level; } int perf_tip(char **strp, const char *dirpath) diff --git a/tools/perf/util/zstd.c b/tools/perf/util/zstd.c index 21a0eb58597c21..d98014902f0124 100644 --- a/tools/perf/util/zstd.c +++ b/tools/perf/util/zstd.c @@ -31,9 +31,11 @@ int zstd_fini(struct zstd_data *data) ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_t dst_size, void *src, size_t src_size, size_t max_record_size, - size_t process_header(void *record, size_t increment)) + ssize_t process_header(void *record, size_t dst_size, + size_t data_size)) { - size_t ret, size, compressed = 0; + size_t ret, compressed = 0; + ssize_t size; ZSTD_inBuffer input = { src, src_size, 0 }; ZSTD_outBuffer output; void *record; @@ -55,12 +57,9 @@ ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_ while (input.pos < input.size) { record = dst; - /* process_header writes the event header into record */ - if (dst_size < sizeof(struct perf_event_header)) - goto reset; - size = process_header(record, 0); + size = process_header(record, dst_size, 0); /* Output buffer full — cannot fit even the record header */ - if (size > dst_size) + if (size < 0) goto reset; compressed += size; dst += size; @@ -74,17 +73,21 @@ ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_ (long)src_size, ZSTD_getErrorName(ret)); goto reset; } - size = output.pos; + compressed += output.pos; + dst += output.pos; + dst_size -= output.pos; /* * No progress: ZSTD couldn't emit any bytes into the * remaining output buffer. Calling process_header - * with size=0 would re-trigger header initialization, + * with output.pos=0 would re-trigger header initialization, * double-subtracting the header size from dst_size and * underflowing the unsigned counter. */ - if (size == 0) + if (output.pos == 0) + goto reset; + size = process_header(record, dst_size, output.pos); + if (size < 0) goto reset; - size = process_header(record, size); compressed += size; dst += size; dst_size -= size; diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h index bcc0b4c84fc09a..09149c32c41c64 100644 --- a/tools/sched_ext/include/scx/compat.bpf.h +++ b/tools/sched_ext/include/scx/compat.bpf.h @@ -84,7 +84,7 @@ bool scx_bpf_dispatch_vtime_from_dsq___old(struct bpf_iter_scx_dsq *it__iter, st * * Compat macro will be dropped on v6.19 release. */ -int bpf_cpumask_populate(struct cpumask *dst, void *src, size_t src__sz) __ksym __weak; +int bpf_cpumask_populate(struct bpf_cpumask *dst, void *src, size_t src__sz) __ksym __weak; #define __COMPAT_bpf_cpumask_populate(cpumask, src, size__sz) \ (bpf_ksym_exists(bpf_cpumask_populate) ? \ diff --git a/tools/sched_ext/scx_flatcg.bpf.c b/tools/sched_ext/scx_flatcg.bpf.c index fec35958182695..ddcf6bc03b11c8 100644 --- a/tools/sched_ext/scx_flatcg.bpf.c +++ b/tools/sched_ext/scx_flatcg.bpf.c @@ -256,7 +256,7 @@ static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) * and thus can't be updated and repositioned. Instead, we collect the * vtime deltas separately and apply it asynchronously here. */ - delta = __sync_fetch_and_sub(&cgc->cvtime_delta, cgc->cvtime_delta); + delta = __sync_fetch_and_and(&cgc->cvtime_delta, 0); cvtime = cgv_node->cvtime + delta; /* @@ -570,7 +570,8 @@ void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) cgc = find_cgrp_ctx(cgrp); if (cgc) { __sync_fetch_and_add(&cgc->cvtime_delta, - p->se.sum_exec_runtime - taskc->bypassed_at); + (p->se.sum_exec_runtime - taskc->bypassed_at) * + FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); taskc->bypassed_at = 0; } bpf_cgroup_release(cgrp); @@ -768,10 +769,18 @@ void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) * cgroup to execute but the latter needs to be done in a loop * and we can't keep the lock held. Oh well... */ + s64 delta = now - cpuc->cur_at - cgrp_slice_ns; + bpf_spin_lock(&cgv_tree_lock); - __sync_fetch_and_add(&cgc->cvtime_delta, - (cpuc->cur_at + cgrp_slice_ns - now) * - FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); + /* keep the dividends positive, BPF division is unsigned */ + if (delta >= 0) + __sync_fetch_and_add(&cgc->cvtime_delta, + (u64)delta * FCG_HWEIGHT_ONE / + (cgc->hweight ?: 1)); + else + __sync_fetch_and_sub(&cgc->cvtime_delta, + (u64)-delta * FCG_HWEIGHT_ONE / + (cgc->hweight ?: 1)); bpf_spin_unlock(&cgv_tree_lock); } else { stat_inc(FCG_STAT_CNS_GONE); diff --git a/tools/sched_ext/scx_flatcg.c b/tools/sched_ext/scx_flatcg.c index de2bef86d64d62..7799782b76d182 100644 --- a/tools/sched_ext/scx_flatcg.c +++ b/tools/sched_ext/scx_flatcg.c @@ -105,12 +105,12 @@ static void fcg_read_stats(struct scx_flatcg *skel, __u64 *stats) __u64 *cnts; __u32 idx; + memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); + cnts = calloc(skel->rodata->nr_cpus, sizeof(__u64)); if (!cnts) return; - memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); - for (idx = 0; idx < FCG_NR_STATS; idx++) { int ret, cpu; diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 0beac1abc58774..fd5883d01d7239 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -495,7 +495,7 @@ static void update_core_sched_head_seq(struct task_struct *p) * moving them to HIGHPRI_DSQ and then consuming them first. This makes minor * difference only when dsp_batch is larger than 1. * - * scx_bpf_dispatch[_vtime]_from_dsq() are allowed both from ops.dispatch() and + * scx_bpf_dsq_move[_vtime]() are allowed both from ops.dispatch() and * non-rq-lock holding BPF programs. As demonstration, this function is called * from qmap_dispatch() and monitor_timerfn(). */ diff --git a/tools/scripts/Makefile.include b/tools/scripts/Makefile.include index 41971a68972dda..7022e78208a236 100644 --- a/tools/scripts/Makefile.include +++ b/tools/scripts/Makefile.include @@ -61,10 +61,18 @@ $(error Invalid value for LLVM, see Documentation/kbuild/llvm.rst) endif $(call allow-override,CC,$(LLVM_PREFIX)clang$(LLVM_SUFFIX)) +$(call allow-override,CLANG,$(LLVM_PREFIX)clang$(LLVM_SUFFIX)) +$(call allow-override,HOSTCC,$(LLVM_PREFIX)clang$(LLVM_SUFFIX)) $(call allow-override,AR,$(LLVM_PREFIX)llvm-ar$(LLVM_SUFFIX)) +$(call allow-override,HOSTAR,$(LLVM_PREFIX)llvm-ar$(LLVM_SUFFIX)) $(call allow-override,LD,$(LLVM_PREFIX)ld.lld$(LLVM_SUFFIX)) +$(call allow-override,HOSTLD,$(LLVM_PREFIX)ld.lld$(LLVM_SUFFIX)) $(call allow-override,CXX,$(LLVM_PREFIX)clang++$(LLVM_SUFFIX)) $(call allow-override,STRIP,$(LLVM_PREFIX)llvm-strip$(LLVM_SUFFIX)) +$(call allow-override,LLVM_STRIP,$(LLVM_PREFIX)llvm-strip$(LLVM_SUFFIX)) +$(call allow-override,LLC,$(LLVM_PREFIX)llc$(LLVM_SUFFIX)) +$(call allow-override,LLVM_CONFIG,$(LLVM_PREFIX)llvm-config$(LLVM_SUFFIX)) +$(call allow-override,LLVM_OBJCOPY,$(LLVM_PREFIX)llvm-objcopy$(LLVM_SUFFIX)) else # Allow setting various cross-compile vars or setting CROSS_COMPILE as a prefix. $(call allow-override,CC,$(CROSS_COMPILE)gcc) @@ -72,26 +80,21 @@ $(call allow-override,AR,$(CROSS_COMPILE)ar) $(call allow-override,LD,$(CROSS_COMPILE)ld) $(call allow-override,CXX,$(CROSS_COMPILE)g++) $(call allow-override,STRIP,$(CROSS_COMPILE)strip) -endif - -CC_NO_CLANG := $(shell $(CC) -dM -E -x c /dev/null | grep -Fq "__clang__"; echo $$?) -ifneq ($(LLVM),) -HOSTAR ?= $(LLVM_PREFIX)llvm-ar$(LLVM_SUFFIX) -HOSTCC ?= $(LLVM_PREFIX)clang$(LLVM_SUFFIX) -HOSTLD ?= $(LLVM_PREFIX)ld.lld$(LLVM_SUFFIX) -else -HOSTAR ?= ar -HOSTCC ?= gcc -HOSTLD ?= ld +# Host versions aren't prefixed +$(call allow-override,HOSTAR,ar) +$(call allow-override,HOSTCC,gcc) +$(call allow-override,HOSTLD,ld) + +# Some tools still require Clang, LLC and/or LLVM utils +$(call allow-override,CLANG,clang) +$(call allow-override,LLC,llc) +$(call allow-override,LLVM_CONFIG,llvm-config) +$(call allow-override,LLVM_OBJCOPY,llvm-objcopy) +$(call allow-override,LLVM_STRIP,llvm-strip) endif -# Some tools require Clang, LLC and/or LLVM utils -CLANG ?= clang -LLC ?= llc -LLVM_CONFIG ?= llvm-config -LLVM_OBJCOPY ?= llvm-objcopy -LLVM_STRIP ?= llvm-strip +CC_NO_CLANG := $(shell $(CC) -dM -E -x c /dev/null | grep -Fq "__clang__"; echo $$?) # Some tools require bpftool SYSTEM_BPFTOOL ?= bpftool diff --git a/tools/testing/cxl/test/cxl.c b/tools/testing/cxl/test/cxl.c index ef92dd35e030ce..a0d75a07613379 100644 --- a/tools/testing/cxl/test/cxl.c +++ b/tools/testing/cxl/test/cxl.c @@ -27,6 +27,7 @@ static bool fail_autoassemble; #define NR_CXL_SWITCH_PORTS 2 #define NR_CXL_PORT_DECODERS 8 #define NR_BRIDGES (NR_CXL_HOST_BRIDGES + NR_CXL_SINGLE_HOST + NR_CXL_RCH) +#define NR_CXL_TYPE2_ACCEL 1 #define MOCK_AUTO_REGION_SIZE_DEFAULT SZ_512M static int mock_auto_region_size = MOCK_AUTO_REGION_SIZE_DEFAULT; @@ -1563,8 +1564,10 @@ static __init int cxl_rch_topo_init(void) struct platform_device *pdev; pdev = platform_device_alloc("cxl_host_bridge", idx); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_bridge; + } mock_companion(adev, &pdev->dev); rc = cxl_mock_platform_device_add(pdev, &cxl_rch[i]); @@ -1618,8 +1621,10 @@ static __init int cxl_single_topo_init(void) pdev = platform_device_alloc("cxl_host_bridge", NR_CXL_HOST_BRIDGES + i); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_bridge; + } mock_companion(adev, &pdev->dev); rc = cxl_mock_platform_device_add(pdev, &cxl_hb_single[i]); @@ -1640,8 +1645,10 @@ static __init int cxl_single_topo_init(void) pdev = platform_device_alloc("cxl_root_port", NR_MULTI_ROOT + i); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_port; + } pdev->dev.parent = &bridge->dev; rc = cxl_mock_platform_device_add(pdev, &cxl_root_single[i]); @@ -1655,8 +1662,10 @@ static __init int cxl_single_topo_init(void) pdev = platform_device_alloc("cxl_switch_uport", NR_MULTI_ROOT + i); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_uport; + } pdev->dev.parent = &root_port->dev; rc = cxl_mock_platform_device_add(pdev, &cxl_swu_single[i]); @@ -1671,8 +1680,10 @@ static __init int cxl_single_topo_init(void) pdev = platform_device_alloc("cxl_switch_dport", i + NR_MEM_MULTI); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_dport; + } pdev->dev.parent = &uport->dev; rc = cxl_mock_platform_device_add(pdev, &cxl_swd_single[i]); @@ -1724,19 +1735,84 @@ static void cxl_single_topo_exit(void) } } -static void cxl_mem_exit(void) +static void cxl_type3_mem_exit(void) { + struct platform_device *pdev; int i; - for (i = ARRAY_SIZE(cxl_rcd) - 1; i >= 0; i--) + for (i = ARRAY_SIZE(cxl_rcd) - 1; i >= 0; i--) { + pdev = cxl_rcd[i]; + if (!pdev) + continue; platform_device_unregister(cxl_rcd[i]); - for (i = ARRAY_SIZE(cxl_mem_single) - 1; i >= 0; i--) + } + + for (i = ARRAY_SIZE(cxl_mem_single) - 1; i >= 0; i--) { + pdev = cxl_mem_single[i]; + if (!pdev) + continue; platform_device_unregister(cxl_mem_single[i]); - for (i = ARRAY_SIZE(cxl_mem) - 1; i >= 0; i--) + } + + for (i = ARRAY_SIZE(cxl_mem) - 1; i >= 0; i--) { + pdev = cxl_mem[i]; + if (!pdev) + continue; + platform_device_unregister(pdev); + } +} + +static void cxl_type2_mem_exit(void) +{ + for (int i = NR_CXL_TYPE2_ACCEL - 1; i >= 0; i--) { + struct platform_device *pdev = cxl_mem[i]; + + if (!pdev) + continue; + platform_device_unregister(pdev); + } +} + +static void cxl_mem_exit(void) +{ + if (type2_test) { + cxl_type2_mem_exit(); + return; + } + + cxl_type3_mem_exit(); +} + +static int cxl_type2_mem_init(void) +{ + int i, rc; + + for (i = 0; i < NR_CXL_TYPE2_ACCEL; i++) { + struct platform_device *dport = cxl_root_port[i]; + struct platform_device *pdev; + + pdev = platform_device_alloc("cxl_type2_accel", i); + if (!pdev) { + rc = -ENOMEM; + goto err_mem; + } + pdev->dev.parent = &dport->dev; + set_dev_node(&pdev->dev, i % 2); + + rc = cxl_mock_platform_device_add(pdev, &cxl_mem[i]); + if (rc) + goto err_mem; + } + + return 0; + +err_mem: + for (i = NR_CXL_TYPE2_ACCEL - 1; i >= 0; i--) platform_device_unregister(cxl_mem[i]); + return rc; } -static int cxl_mem_init(void) +static int cxl_type3_mem_init(void) { int i, rc; @@ -1745,8 +1821,10 @@ static int cxl_mem_init(void) struct platform_device *pdev; pdev = platform_device_alloc("cxl_mem", i); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_mem; + } pdev->dev.parent = &dport->dev; set_dev_node(&pdev->dev, i % 2); @@ -1760,8 +1838,10 @@ static int cxl_mem_init(void) struct platform_device *pdev; pdev = platform_device_alloc("cxl_mem", NR_MEM_MULTI + i); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_single; + } pdev->dev.parent = &dport->dev; set_dev_node(&pdev->dev, i % 2); @@ -1776,8 +1856,10 @@ static int cxl_mem_init(void) struct platform_device *pdev; pdev = platform_device_alloc("cxl_rcd", idx); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_rcd; + } pdev->dev.parent = &rch->dev; set_dev_node(&pdev->dev, i % 2); @@ -1800,6 +1882,13 @@ err_mem: return rc; } +static int cxl_mem_init(void) +{ + if (type2_test) + return cxl_type2_mem_init(); + return cxl_type3_mem_init(); +} + static ssize_t decoder_reset_preserve_registry_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -1827,58 +1916,48 @@ static struct attribute *cxl_acpi_attrs[] = { }; ATTRIBUTE_GROUPS(cxl_acpi); -static __init int cxl_test_init(void) +static bool __init have_multiple_modparms(void) { - int rc, i; - struct range mappable; + int count = 0; - if (!IS_ALIGNED(mock_auto_region_size, PMD_SIZE)) { - pr_err_once("mock_auto_region_size %d must be PMD-aligned\n", - mock_auto_region_size); - return -EINVAL; - } + if (interleave_arithmetic) + count++; + if (extended_linear_cache) + count++; + if (hmem_test) + count++; - cxl_acpi_test(); - cxl_core_test(); - cxl_mem_test(); - cxl_pmem_test(); - cxl_port_test(); + return count > 1; +} - register_cxl_mock_ops(&cxl_mock_ops); +static void host_bridges_remove(void) +{ + int i; - cxl_mock_pool = gen_pool_create(ilog2(SZ_2M), NUMA_NO_NODE); - if (!cxl_mock_pool) { - rc = -ENOMEM; - goto err_gen_pool_create; - } - mappable = mhp_get_pluggable_range(true); + for (i = ARRAY_SIZE(cxl_host_bridge) - 1; i >= 0; i--) { + struct platform_device *pdev = cxl_host_bridge[i]; - rc = gen_pool_add(cxl_mock_pool, - min(iomem_resource.end + 1 - SZ_64G, - mappable.end + 1 - SZ_64G), - SZ_64G, NUMA_NO_NODE); - if (rc) - goto err_gen_pool_add; + if (!pdev) + continue; - if (interleave_arithmetic == 1) { - cfmws_start = CFMWS_XOR_ARRAY_START; - cfmws_end = CFMWS_XOR_ARRAY_END; - } else { - cfmws_start = CFMWS_MOD_ARRAY_START; - cfmws_end = CFMWS_MOD_ARRAY_END; + sysfs_remove_link(&pdev->dev.kobj, "physical_node"); + platform_device_unregister(cxl_host_bridge[i]); } +} - rc = populate_cedt(); - if (rc) - goto err_populate; +static int host_bridges_populate(void) +{ + int rc = 0; - for (i = 0; i < ARRAY_SIZE(cxl_host_bridge); i++) { + for (int i = 0; i < ARRAY_SIZE(cxl_host_bridge); i++) { struct acpi_device *adev = &host_bridge[i]; struct platform_device *pdev; pdev = platform_device_alloc("cxl_host_bridge", i); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_bridge; + } mock_companion(adev, &pdev->dev); rc = cxl_mock_platform_device_add(pdev, &cxl_host_bridge[i]); @@ -1892,14 +1971,40 @@ static __init int cxl_test_init(void) goto err_bridge; } - for (i = 0; i < ARRAY_SIZE(cxl_root_port); i++) { + return 0; + +err_bridge: + host_bridges_remove(); + return rc; +} + +static void cxl_rootports_remove(void) +{ + for (int i = ARRAY_SIZE(cxl_root_port) - 1; i >= 0; i--) { + struct platform_device *pdev = cxl_root_port[i]; + + if (!pdev) + continue; + + platform_device_unregister(pdev); + } +} + +static int cxl_rootports_populate(void) +{ + int rc = 0; + + for (int i = 0; i < ARRAY_SIZE(cxl_root_port); i++) { struct platform_device *bridge = cxl_host_bridge[i % ARRAY_SIZE(cxl_host_bridge)]; struct platform_device *pdev; pdev = platform_device_alloc("cxl_root_port", i); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_port; + } + pdev->dev.parent = &bridge->dev; rc = cxl_mock_platform_device_add(pdev, &cxl_root_port[i]); @@ -1907,14 +2012,39 @@ static __init int cxl_test_init(void) goto err_port; } - BUILD_BUG_ON(ARRAY_SIZE(cxl_switch_uport) != ARRAY_SIZE(cxl_root_port)); - for (i = 0; i < ARRAY_SIZE(cxl_switch_uport); i++) { + return 0; + +err_port: + cxl_rootports_remove(); + return rc; +} + +static void cxl_usps_remove(void) +{ + for (int i = ARRAY_SIZE(cxl_switch_uport) - 1; i >= 0; i--) { + struct platform_device *pdev = cxl_switch_uport[i]; + + if (!pdev) + continue; + + platform_device_unregister(cxl_switch_uport[i]); + } +} + +static int cxl_usps_populate(void) +{ + int rc = 0; + + for (int i = 0; i < ARRAY_SIZE(cxl_switch_uport); i++) { struct platform_device *root_port = cxl_root_port[i]; struct platform_device *pdev; pdev = platform_device_alloc("cxl_switch_uport", i); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_uport; + } + pdev->dev.parent = &root_port->dev; rc = cxl_mock_platform_device_add(pdev, &cxl_switch_uport[i]); @@ -1922,14 +2052,40 @@ static __init int cxl_test_init(void) goto err_uport; } - for (i = 0; i < ARRAY_SIZE(cxl_switch_dport); i++) { + return 0; + +err_uport: + cxl_usps_remove(); + return rc; +} + +static void cxl_dsps_remove(void) +{ + for (int i = ARRAY_SIZE(cxl_switch_dport) - 1; i >= 0; i--) { + struct platform_device *pdev = cxl_switch_dport[i]; + + if (!pdev) + continue; + + platform_device_unregister(cxl_switch_dport[i]); + } +} + + +static int cxl_dsps_populate(void) +{ + int rc = 0; + + for (int i = 0; i < ARRAY_SIZE(cxl_switch_dport); i++) { struct platform_device *uport = cxl_switch_uport[i % ARRAY_SIZE(cxl_switch_uport)]; struct platform_device *pdev; pdev = platform_device_alloc("cxl_switch_dport", i); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_dport; + } pdev->dev.parent = &uport->dev; rc = cxl_mock_platform_device_add(pdev, &cxl_switch_dport[i]); @@ -1937,17 +2093,181 @@ static __init int cxl_test_init(void) goto err_dport; } + return 0; + +err_dport: + cxl_dsps_remove(); + return rc; +} + +static void cxl_switches_remove(void) +{ + cxl_dsps_remove(); + cxl_usps_remove(); +} + +static int cxl_switches_populate(void) +{ + int rc; + + BUILD_BUG_ON(ARRAY_SIZE(cxl_switch_uport) != ARRAY_SIZE(cxl_root_port)); + rc = cxl_usps_populate(); + if (rc) + return rc; + + rc = cxl_dsps_populate(); + if (rc) { + cxl_usps_remove(); + return rc; + } + + return 0; +} + +static void cxl_type2_topo_exit(void) +{ + cxl_rootports_remove(); + host_bridges_remove(); +} + +static int cxl_type2_topo_init(void) +{ + int rc; + + rc = host_bridges_populate(); + if (rc) + return rc; + + rc = cxl_rootports_populate(); + if (rc) { + host_bridges_remove(); + return rc; + } + + return 0; +} + +static void cxl_type3_topo_exit(void) +{ + cxl_rch_topo_exit(); + cxl_single_topo_exit(); + cxl_switches_remove(); + cxl_rootports_remove(); + host_bridges_remove(); +} + +static int cxl_type3_topo_init(void) +{ + int rc; + + rc = host_bridges_populate(); + if (rc) + return rc; + + rc = cxl_rootports_populate(); + if (rc) + goto err_host_bridges; + + rc = cxl_switches_populate(); + if (rc) + goto err_root_ports; + rc = cxl_single_topo_init(); if (rc) - goto err_dport; + goto err_switches; rc = cxl_rch_topo_init(); if (rc) goto err_single; + return 0; + +err_single: + cxl_single_topo_exit(); +err_switches: + cxl_switches_remove(); +err_root_ports: + cxl_rootports_remove(); +err_host_bridges: + host_bridges_remove(); + return rc; +} + +static void cxl_topo_exit(void) +{ + if (type2_test) { + cxl_type2_topo_exit(); + return; + } + + cxl_type3_topo_exit(); +} + +static int cxl_topo_init(void) +{ + if (type2_test) + return cxl_type2_topo_init(); + return cxl_type3_topo_init(); +} + +static __init int cxl_test_init(void) +{ + struct range mappable; + int rc; + + /* Enforce a single module param active at a time */ + if (have_multiple_modparms()) + return -EINVAL; + + if (!IS_ALIGNED(mock_auto_region_size, PMD_SIZE)) { + pr_err_once("mock_auto_region_size %d must be PMD-aligned\n", + mock_auto_region_size); + return -EINVAL; + } + + cxl_acpi_test(); + cxl_core_test(); + cxl_mem_test(); + cxl_pmem_test(); + cxl_port_test(); + + register_cxl_mock_ops(&cxl_mock_ops); + + cxl_mock_pool = gen_pool_create(ilog2(SZ_2M), NUMA_NO_NODE); + if (!cxl_mock_pool) { + rc = -ENOMEM; + goto err_gen_pool_create; + } + mappable = mhp_get_pluggable_range(true); + + rc = gen_pool_add(cxl_mock_pool, + min(iomem_resource.end + 1 - SZ_64G, + mappable.end + 1 - SZ_64G), + SZ_64G, NUMA_NO_NODE); + if (rc) + goto err_gen_pool_add; + + if (interleave_arithmetic == 1) { + cfmws_start = CFMWS_XOR_ARRAY_START; + cfmws_end = CFMWS_XOR_ARRAY_END; + } else { + cfmws_start = CFMWS_MOD_ARRAY_START; + cfmws_end = CFMWS_MOD_ARRAY_END; + } + + rc = populate_cedt(); + if (rc) + goto err_populate; + + rc = cxl_topo_init(); + if (rc) + goto err_populate; + cxl_acpi = platform_device_alloc("cxl_acpi", 0); - if (!cxl_acpi) - goto err_rch; + if (!cxl_acpi) { + rc = -ENOMEM; + goto err_topo; + } mock_companion(&acpi0017_mock, &cxl_acpi->dev); acpi0017_mock.dev.bus = &platform_bus_type; @@ -1955,7 +2275,7 @@ static __init int cxl_test_init(void) rc = cxl_mock_platform_device_add(cxl_acpi, NULL); if (rc) - goto err_rch; + goto err_topo; rc = cxl_mem_init(); if (rc) @@ -1971,28 +2291,8 @@ err_mem: cxl_mem_exit(); err_root: platform_device_unregister(cxl_acpi); -err_rch: - cxl_rch_topo_exit(); -err_single: - cxl_single_topo_exit(); -err_dport: - for (i = ARRAY_SIZE(cxl_switch_dport) - 1; i >= 0; i--) - platform_device_unregister(cxl_switch_dport[i]); -err_uport: - for (i = ARRAY_SIZE(cxl_switch_uport) - 1; i >= 0; i--) - platform_device_unregister(cxl_switch_uport[i]); -err_port: - for (i = ARRAY_SIZE(cxl_root_port) - 1; i >= 0; i--) - platform_device_unregister(cxl_root_port[i]); -err_bridge: - for (i = ARRAY_SIZE(cxl_host_bridge) - 1; i >= 0; i--) { - struct platform_device *pdev = cxl_host_bridge[i]; - - if (!pdev) - continue; - sysfs_remove_link(&pdev->dev.kobj, "physical_node"); - platform_device_unregister(cxl_host_bridge[i]); - } +err_topo: + cxl_topo_exit(); err_populate: depopulate_all_mock_resources(); err_gen_pool_add: @@ -2015,27 +2315,10 @@ static void free_decoder_registry(void) static __exit void cxl_test_exit(void) { - int i; - hmem_test_exit(); cxl_mem_exit(); platform_device_unregister(cxl_acpi); - cxl_rch_topo_exit(); - cxl_single_topo_exit(); - for (i = ARRAY_SIZE(cxl_switch_dport) - 1; i >= 0; i--) - platform_device_unregister(cxl_switch_dport[i]); - for (i = ARRAY_SIZE(cxl_switch_uport) - 1; i >= 0; i--) - platform_device_unregister(cxl_switch_uport[i]); - for (i = ARRAY_SIZE(cxl_root_port) - 1; i >= 0; i--) - platform_device_unregister(cxl_root_port[i]); - for (i = ARRAY_SIZE(cxl_host_bridge) - 1; i >= 0; i--) { - struct platform_device *pdev = cxl_host_bridge[i]; - - if (!pdev) - continue; - sysfs_remove_link(&pdev->dev.kobj, "physical_node"); - platform_device_unregister(cxl_host_bridge[i]); - } + cxl_topo_exit(); depopulate_all_mock_resources(); gen_pool_destroy(cxl_mock_pool); unregister_cxl_mock_ops(&cxl_mock_ops); diff --git a/tools/testing/cxl/test/hmem_test.c b/tools/testing/cxl/test/hmem_test.c index 3a1a089e1721be..0fa00f7e16db57 100644 --- a/tools/testing/cxl/test/hmem_test.c +++ b/tools/testing/cxl/test/hmem_test.c @@ -3,8 +3,9 @@ #include #include #include "../../../drivers/dax/bus.h" +#include "mock.h" -static bool hmem_test; +bool hmem_test; static void hmem_test_work(struct work_struct *work) { diff --git a/tools/testing/cxl/test/mock.h b/tools/testing/cxl/test/mock.h index 4f57dc80ae7d5f..846d7c5d6eaa94 100644 --- a/tools/testing/cxl/test/mock.h +++ b/tools/testing/cxl/test/mock.h @@ -5,6 +5,8 @@ #include #include +extern bool hmem_test; + struct cxl_mock_ops { struct list_head list; bool (*is_mock_adev)(struct acpi_device *dev); diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index ac3f7159e67fea..91d234ac3b5792 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -126,7 +126,7 @@ def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) lines.pop() # Filter out any extraneous non-test output that might have gotten mixed in. - return [l for l in output if re.match(r'^[^\s.]+\.[^\s.]+$', l)] + return [l for l in lines if re.match(r'^[^\s.]+\.[^\s.]+$', l)] def _list_tests_attr(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> Iterable[str]: args = ['kunit.action=list_attr'] diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index da88c3a1651d09..85ae21754bdf90 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -979,6 +979,18 @@ class KUnitMainTest(unittest.TestCase): self.linux_source_mock.run_kernel.assert_called_once_with( args=['kunit.action=list'], build_dir='.kunit', filter_glob='suite*', filter='', filter_action=None, timeout=300) + def test_list_tests_with_prefix(self): + want = ['suite.test1', 'suite.test2', 'suite2.test1'] + self.linux_source_mock.run_kernel.return_value = [ + '[ 0.100000] TAP version 14', + '[ 0.200000] suite.test1', + '[ 0.200000] suite.test2', + '[ 0.300000] suite2.test1'] + + got = kunit._list_tests(self.linux_source_mock, + kunit.KunitExecRequest(None, None, None, False, False, '.kunit', 300, 'suite*', '', None, None, 'suite', False, False, False)) + self.assertEqual(got, want) + @mock.patch.object(kunit, '_list_tests') def test_run_isolated_by_suite(self, mock_tests): mock_tests.return_value = ['suite.test1', 'suite.test2', 'suite2.test1'] diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.c b/tools/testing/selftests/arm64/fp/fp-ptrace.c index 22c584b78be519..b435837c8c0e87 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace.c +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.c @@ -65,6 +65,9 @@ /* VL 128..2048 in powers of 2 */ #define MAX_NUM_VLS 5 +/* Sentinel for detecting buffer bytes the kernel did not write */ +#define REGSET_SENTINEL 0xa5 + /* * FPMR bits we can set without doing feature checks to see if values * are valid. @@ -181,6 +184,20 @@ static bool compare_buffer(const char *name, void *out, return false; } +static bool buffer_is_filled(const void *buffer, size_t size, + unsigned char value) +{ + const unsigned char *bytes = buffer; + size_t i; + + for (i = 0; i < size; i++) { + if (bytes[i] != value) + return false; + } + + return true; +} + struct test_config { int sve_vl_in; int sve_vl_expected; @@ -401,6 +418,7 @@ static bool check_ptrace_values_sve(pid_t child, struct test_config *config) struct user_sve_header *sve; struct user_fpsimd_state *fpsimd; struct iovec iov; + size_t buf_size; int ret, vq; bool pass = true; @@ -409,14 +427,16 @@ static bool check_ptrace_values_sve(pid_t child, struct test_config *config) vq = __sve_vq_from_vl(config->sve_vl_in); - iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); - iov.iov_base = malloc(iov.iov_len); + buf_size = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); + iov.iov_len = buf_size; + iov.iov_base = malloc(buf_size); if (!iov.iov_base) { ksft_print_msg("OOM allocating %lu byte SVE buffer\n", iov.iov_len); return false; } + memset(iov.iov_base, REGSET_SENTINEL, buf_size); ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_SVE, &iov); if (ret != 0) { ksft_print_msg("Failed to read initial SVE: %s (%d)\n", @@ -440,10 +460,16 @@ static bool check_ptrace_values_sve(pid_t child, struct test_config *config) } if (svcr_in & SVCR_SM) { - if (sve->size != sizeof(sve)) { + if (sve->size != sizeof(*sve)) { ksft_print_msg("NT_ARM_SVE reports data with PSTATE.SM\n"); pass = false; } + if (!buffer_is_filled(iov.iov_base + sizeof(*sve), + buf_size - sizeof(*sve), REGSET_SENTINEL)) { + ksft_print_msg("NT_ARM_SVE wrote beyond its header with PSTATE.SM\n"); + pass = false; + } + goto out; } else { if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { ksft_print_msg("Mismatch in SVE header size: %d != %lu\n", @@ -485,6 +511,7 @@ static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) struct user_sve_header *sve; struct user_fpsimd_state *fpsimd; struct iovec iov; + size_t buf_size; int ret, vq; bool pass = true; @@ -493,14 +520,16 @@ static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) vq = __sve_vq_from_vl(config->sme_vl_in); - iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); - iov.iov_base = malloc(iov.iov_len); + buf_size = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); + iov.iov_len = buf_size; + iov.iov_base = malloc(buf_size); if (!iov.iov_base) { ksft_print_msg("OOM allocating %lu byte SSVE buffer\n", iov.iov_len); return false; } + memset(iov.iov_base, REGSET_SENTINEL, buf_size); ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_SSVE, &iov); if (ret != 0) { ksft_print_msg("Failed to read initial SSVE: %s (%d)\n", @@ -523,10 +552,16 @@ static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) } if (!(svcr_in & SVCR_SM)) { - if (sve->size != sizeof(sve)) { + if (sve->size != sizeof(*sve)) { ksft_print_msg("NT_ARM_SSVE reports data without PSTATE.SM\n"); pass = false; } + if (!buffer_is_filled(iov.iov_base + sizeof(*sve), + buf_size - sizeof(*sve), REGSET_SENTINEL)) { + ksft_print_msg("NT_ARM_SSVE wrote beyond its header without PSTATE.SM\n"); + pass = false; + } + goto out; } else { if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { ksft_print_msg("Mismatch in SSVE header size: %d != %lu\n", diff --git a/tools/testing/selftests/arm64/fp/sve-test.S b/tools/testing/selftests/arm64/fp/sve-test.S index 80e072f221cdee..7ef7835389e768 100644 --- a/tools/testing/selftests/arm64/fp/sve-test.S +++ b/tools/testing/selftests/arm64/fp/sve-test.S @@ -298,15 +298,20 @@ function irritator_handler add x0, x0, #1 str x0, [x2, #ucontext_regs + 8 * 23] +#ifndef SSVE // Corrupt some random Z-regs movi v0.8b, #1 movi v9.16b, #2 movi v31.8b, #3 // And P0 ptrue p0.d -#ifndef SSVE // And FFR wrffr p15.b +#else + // Enter and exit streaming mode, will reset all of the V, Z, P + // and FFR registers that the system has. + smstart_sm + smstop #endif ret diff --git a/tools/testing/selftests/arm64/mte/check_buffer_fill.c b/tools/testing/selftests/arm64/mte/check_buffer_fill.c index ff4e0750334936..039b1d7d856637 100644 --- a/tools/testing/selftests/arm64/mte/check_buffer_fill.c +++ b/tools/testing/selftests/arm64/mte/check_buffer_fill.c @@ -406,6 +406,8 @@ int main(int argc, char *argv[]) size_t page_size = getpagesize(); int item = ARRAY_SIZE(sizes); + ksft_print_header(); + sizes[item - 3] = page_size - 1; sizes[item - 2] = page_size; sizes[item - 1] = page_size + 1; diff --git a/tools/testing/selftests/arm64/mte/check_child_memory.c b/tools/testing/selftests/arm64/mte/check_child_memory.c index 5e97ee792e4d2e..e6a8acca2a9411 100644 --- a/tools/testing/selftests/arm64/mte/check_child_memory.c +++ b/tools/testing/selftests/arm64/mte/check_child_memory.c @@ -146,6 +146,8 @@ int main(int argc, char *argv[]) int err; int item = ARRAY_SIZE(sizes); + ksft_print_header(); + page_size = getpagesize(); if (!page_size) { ksft_print_msg("ERR: Unable to get page size\n"); diff --git a/tools/testing/selftests/arm64/mte/check_gcr_el1_cswitch.c b/tools/testing/selftests/arm64/mte/check_gcr_el1_cswitch.c index 325bca0de0f6e1..d23f154d3288c5 100644 --- a/tools/testing/selftests/arm64/mte/check_gcr_el1_cswitch.c +++ b/tools/testing/selftests/arm64/mte/check_gcr_el1_cswitch.c @@ -131,6 +131,7 @@ int main(int argc, char *argv[]) if (err) return err; + ksft_print_header(); ksft_set_plan(1); evaluate_test(mte_gcr_fork_test(), diff --git a/tools/testing/selftests/arm64/mte/check_hugetlb_options.c b/tools/testing/selftests/arm64/mte/check_hugetlb_options.c index aad1234c7e0feb..23e4a7a9950c1f 100644 --- a/tools/testing/selftests/arm64/mte/check_hugetlb_options.c +++ b/tools/testing/selftests/arm64/mte/check_hugetlb_options.c @@ -230,6 +230,8 @@ int main(int argc, char *argv[]) void *map_ptr; unsigned long map_size; + ksft_print_header(); + err = mte_default_setup(); if (err) return err; diff --git a/tools/testing/selftests/arm64/mte/check_ksm_options.c b/tools/testing/selftests/arm64/mte/check_ksm_options.c index 0cf5faef172481..4855b737d5507a 100644 --- a/tools/testing/selftests/arm64/mte/check_ksm_options.c +++ b/tools/testing/selftests/arm64/mte/check_ksm_options.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +23,20 @@ static size_t page_sz; static unsigned long ksm_sysfs[5]; +static bool has_merge_across_nodes; + +static bool merge_across_nodes_available(void) +{ + const char *path = PATH_KSM "merge_across_nodes"; + + if (!access(path, R_OK | W_OK)) + return true; + if (errno == ENOENT) + return false; + + ksft_exit_skip("Unable to read and write %s: %s\n", path, + strerror(errno)); +} static unsigned long read_sysfs(char *str) { @@ -56,8 +71,10 @@ static void write_sysfs(char *str, unsigned long val) static void mte_ksm_setup(void) { - ksm_sysfs[0] = read_sysfs(PATH_KSM "merge_across_nodes"); - write_sysfs(PATH_KSM "merge_across_nodes", 1); + if (has_merge_across_nodes) { + ksm_sysfs[0] = read_sysfs(PATH_KSM "merge_across_nodes"); + write_sysfs(PATH_KSM "merge_across_nodes", 1); + } ksm_sysfs[1] = read_sysfs(PATH_KSM "sleep_millisecs"); write_sysfs(PATH_KSM "sleep_millisecs", 0); ksm_sysfs[2] = read_sysfs(PATH_KSM "run"); @@ -70,7 +87,8 @@ static void mte_ksm_setup(void) static void mte_ksm_restore(void) { - write_sysfs(PATH_KSM "merge_across_nodes", ksm_sysfs[0]); + if (has_merge_across_nodes) + write_sysfs(PATH_KSM "merge_across_nodes", ksm_sysfs[0]); write_sysfs(PATH_KSM "sleep_millisecs", ksm_sysfs[1]); write_sysfs(PATH_KSM "run", ksm_sysfs[2]); write_sysfs(PATH_KSM "max_page_sharing", ksm_sysfs[3]); @@ -132,9 +150,16 @@ int main(int argc, char *argv[]) { int err; + ksft_print_header(); + err = mte_default_setup(); if (err) return err; + + if (geteuid() != 0) + ksft_exit_skip("Please run the test as root\n"); + + has_merge_across_nodes = merge_across_nodes_available(); page_sz = getpagesize(); if (!page_sz) { ksft_print_msg("ERR: Unable to get page size\n"); diff --git a/tools/testing/selftests/arm64/mte/check_mmap_options.c b/tools/testing/selftests/arm64/mte/check_mmap_options.c index c100af3012cbb9..492f2cd41f43b7 100644 --- a/tools/testing/selftests/arm64/mte/check_mmap_options.c +++ b/tools/testing/selftests/arm64/mte/check_mmap_options.c @@ -945,6 +945,8 @@ int main(int argc, char *argv[]) }, }; + ksft_print_header(); + err = mte_default_setup(); if (err) return err; diff --git a/tools/testing/selftests/arm64/mte/check_prctl.c b/tools/testing/selftests/arm64/mte/check_prctl.c index f7f320defa7b98..d16a91117eef9f 100644 --- a/tools/testing/selftests/arm64/mte/check_prctl.c +++ b/tools/testing/selftests/arm64/mte/check_prctl.c @@ -119,7 +119,7 @@ int main(void) int i; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(mte_modes)); + ksft_set_plan(ARRAY_SIZE(mte_modes) + 1); check_basic_read(); for (i = 0; i < ARRAY_SIZE(mte_modes); i++) diff --git a/tools/testing/selftests/arm64/mte/check_tags_inclusion.c b/tools/testing/selftests/arm64/mte/check_tags_inclusion.c index 4b764f2a818576..6b4fa6705d7c1b 100644 --- a/tools/testing/selftests/arm64/mte/check_tags_inclusion.c +++ b/tools/testing/selftests/arm64/mte/check_tags_inclusion.c @@ -175,6 +175,8 @@ int main(int argc, char *argv[]) { int err; + ksft_print_header(); + err = mte_default_setup(); if (err) return err; diff --git a/tools/testing/selftests/arm64/mte/check_user_mem.c b/tools/testing/selftests/arm64/mte/check_user_mem.c index fb7936c4e09785..af343aa617323d 100644 --- a/tools/testing/selftests/arm64/mte/check_user_mem.c +++ b/tools/testing/selftests/arm64/mte/check_user_mem.c @@ -201,6 +201,8 @@ int main(int argc, char *argv[]) int tag_offsets[] = {page_sz, MT_GRANULE_SIZE}; char test_name[TEST_NAME_MAX]; + ksft_print_header(); + page_sz = getpagesize(); if (!page_sz) { ksft_print_msg("ERR: Unable to get page size\n"); diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index b642ee489ea640..5ae24ee101007a 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -324,8 +324,6 @@ TRUNNER_BPFTOOL := $(DEFAULT_BPFTOOL) USE_BOOTSTRAP := "bootstrap/" endif -TEST_GEN_PROGS_EXTENDED += $(TRUNNER_BPFTOOL) - $(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED): $(BPFOBJ) TESTING_HELPERS := $(OUTPUT)/testing_helpers.o @@ -466,7 +464,7 @@ endif CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH)) BPF_CFLAGS = -g -Wall -Werror -D__TARGET_ARCH_$(SRCARCH) $(MENDIAN) \ -I$(INCLUDE_DIR) -I$(CURDIR) -I$(APIDIR) \ - -I$(CURDIR)/libarena/include \ + -I$(TOOLSINCDIR) -I$(CURDIR)/libarena/include \ -I$(abspath $(OUTPUT)/../usr/include) \ -std=gnu11 \ -fno-strict-aliasing \ @@ -1055,10 +1053,13 @@ endif DEFAULT_INSTALL_RULE := $(INSTALL_RULE) override define INSTALL_RULE $(DEFAULT_INSTALL_RULE) + @mkdir -p $(INSTALL_PATH)/tools/sbin + @rsync -a $(if $(PERMISSIVE),--ignore-missing-args) $(TRUNNER_BPFTOOL) $(INSTALL_PATH)/tools/sbin/ + @rsync -a $(if $(PERMISSIVE),--ignore-missing-args) $(OUTPUT)/*.BTF $(INSTALL_PATH)/ @for DIR in $(TEST_INST_SUBDIRS); do \ mkdir -p $(INSTALL_PATH)/$$DIR; \ rsync -a $(if $(PERMISSIVE),--ignore-missing-args) \ - $(OUTPUT)/$$DIR/*.bpf.o \ + $(OUTPUT)/$$DIR/*.bpf.o $(OUTPUT)/$$DIR/*.BTF \ $(INSTALL_PATH)/$$DIR; \ done endef diff --git a/tools/testing/selftests/bpf/README.rst b/tools/testing/selftests/bpf/README.rst index 37164322a1023f..07c834433b3889 100644 --- a/tools/testing/selftests/bpf/README.rst +++ b/tools/testing/selftests/bpf/README.rst @@ -107,12 +107,12 @@ Docker container and local rootfs image. The overall steps are as follows: tools/testing/selftests/bpf/vmtest.sh \ -l -- \ ./test_progs -d \ - \"$(cat tools/testing/selftests/bpf/DENYLIST.riscv64 \ + "$(cat tools/testing/selftests/bpf/DENYLIST.riscv64 \ | cut -d'#' -f1 \ | sed -e 's/^[[:space:]]*//' \ -e 's/[[:space:]]*$//' \ | tr -s '\n' ',' \ - )\" + )" Link: https://github.com/pulehui/riscv-bpf-vmtest.git [0] Link: https://github.com/libbpf/ci/blob/main/rootfs/mkrootfs_debian.sh [1] diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h b/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h index fc27a4bcf5d7ef..b6676dd67bc09c 100644 --- a/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h +++ b/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h @@ -115,7 +115,7 @@ static inline int libarena_asan_init(int arena_asan_init_fd, { LIBBPF_OPTS(bpf_test_run_opts, opts); struct asan_init_args args; - u64 globals_pages; + u64 globals_pages = 0; int ret; ret = libarena_get_globals_pages(arena_asan_init_fd, diff --git a/tools/testing/selftests/bpf/libarena/src/common.bpf.c b/tools/testing/selftests/bpf/libarena/src/common.bpf.c index 50be57213dfb8d..99553aac3d4338 100644 --- a/tools/testing/selftests/bpf/libarena/src/common.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/common.bpf.c @@ -38,6 +38,12 @@ __weak int arena_buddy_reset(void) return buddy_init(&buddy); } +SEC("syscall") +__weak int arena_buddy_destroy(void) +{ + return buddy_destroy(&buddy); +} + __weak void __arena *arena_malloc(size_t size) { return buddy_alloc(&buddy, size); diff --git a/tools/testing/selftests/bpf/network_helpers.c b/tools/testing/selftests/bpf/network_helpers.c index b82f572641b7db..db935a9d9fc1a1 100644 --- a/tools/testing/selftests/bpf/network_helpers.c +++ b/tools/testing/selftests/bpf/network_helpers.c @@ -111,7 +111,7 @@ int start_server_addr(int type, const struct sockaddr_storage *addr, socklen_t a if (settimeo(fd, opts->timeout_ms)) goto error_close; - if (type == SOCK_STREAM && + if ((type & SOCK_TYPE_MASK) == SOCK_STREAM && setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on))) { log_err("Failed to enable SO_REUSEADDR"); goto error_close; @@ -128,7 +128,7 @@ int start_server_addr(int type, const struct sockaddr_storage *addr, socklen_t a goto error_close; } - if (type == SOCK_STREAM) { + if ((type & SOCK_TYPE_MASK) == SOCK_STREAM) { if (listen(fd, opts->backlog ? MAX(opts->backlog, 0) : 1) < 0) { log_err("Failed to listed on socket"); goto error_close; diff --git a/tools/testing/selftests/bpf/network_helpers.h b/tools/testing/selftests/bpf/network_helpers.h index 79a010c88e11c8..75133119c04a35 100644 --- a/tools/testing/selftests/bpf/network_helpers.h +++ b/tools/testing/selftests/bpf/network_helpers.h @@ -25,6 +25,11 @@ typedef __u16 __sum16; #define VIP_NUM 5 #define MAGIC_BYTES 123 +/* include/linux/net.h */ +#ifndef SOCK_TYPE_MASK +#define SOCK_TYPE_MASK 0xf +#endif + struct network_helper_opts { int timeout_ms; int proto; diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c b/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c index b7c18d590b9938..5a1e08d39a06b4 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c @@ -10,6 +10,17 @@ #include "cgroup_iter_memcg.h" #include "cgroup_iter_memcg.skel.h" +/* + * memcg stats are cached per-cpu and only become visible once the periodic + * flusher runs (FLUSH_TIME, 2s), or once pending updates cross + * MEMCG_CHARGE_BATCH * num_online_cpus(). That threshold grows with the CPU + * count, so on a large machine a single pass does not reach it and + * bpf_mem_cgroup_flush_stats() returns without flushing anything. Retry for + * long enough to cover a flusher cycle. + */ +#define MEMCG_STAT_RETRIES 16 +#define MEMCG_STAT_RETRY_DELAY_US (250 * 1000) + static int read_stats(struct bpf_link *link) { int fd, ret = 0; @@ -35,11 +46,13 @@ static int read_stats(struct bpf_link *link) static void test_anon(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; void *map; size_t len; len = sysconf(_SC_PAGESIZE) * 1024; +retry: /* * Increase memcg anon usage by mapping and writing * to a new anon region. @@ -53,6 +66,12 @@ static void test_anon(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup; + if (!memcg_query->nr_anon_mapped && ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + munmap(map, len); + goto retry; + } + ASSERT_GT(memcg_query->nr_anon_mapped, 0, "final anon mapped val"); cleanup: @@ -61,6 +80,7 @@ cleanup: static void test_file(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; void *map; size_t len; char *path; @@ -76,6 +96,7 @@ static void test_file(struct bpf_link *link, struct memcg_query *memcg_query) fd = open(path, O_CREAT | O_RDWR, 0644); if (!ASSERT_OK_FD(fd, "open fd")) return; +retry: if (!ASSERT_OK(ftruncate(fd, len), "ftruncate")) goto cleanup_fd; @@ -88,6 +109,13 @@ static void test_file(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup_map; + if ((!memcg_query->nr_file_pages || !memcg_query->nr_file_mapped) && + ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + munmap(map, len); + goto retry; + } + ASSERT_GT(memcg_query->nr_file_pages, 0, "final file value"); ASSERT_GT(memcg_query->nr_file_mapped, 0, "final file mapped value"); @@ -100,6 +128,7 @@ cleanup_fd: static void test_shmem(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; size_t len; int fd; @@ -113,12 +142,18 @@ static void test_shmem(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK_FD(fd, "memfd_create")) return; +retry: if (!ASSERT_OK(fallocate(fd, 0, 0, len), "fallocate")) goto cleanup; if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup; + if (!memcg_query->nr_shmem && ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + goto retry; + } + ASSERT_GT(memcg_query->nr_shmem, 0, "final shmem value"); cleanup: @@ -127,11 +162,13 @@ cleanup: static void test_pgfault(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; void *map; size_t len; len = sysconf(_SC_PAGESIZE) * 1024; +retry: /* Create region to use for triggering a page fault. */ map = mmap(NULL, len, PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); if (!ASSERT_NEQ(map, MAP_FAILED, "mmap anon")) @@ -143,6 +180,12 @@ static void test_pgfault(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup; + if (!memcg_query->pgfault && ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + munmap(map, len); + goto retry; + } + ASSERT_GT(memcg_query->pgfault, 0, "final pgfault val"); cleanup: diff --git a/tools/testing/selftests/bpf/prog_tests/global_map_resize.c b/tools/testing/selftests/bpf/prog_tests/global_map_resize.c index 56b5baef35c8c3..602ce30f1720c6 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_map_resize.c +++ b/tools/testing/selftests/bpf/prog_tests/global_map_resize.c @@ -23,6 +23,7 @@ static void global_map_resize_bss_subtest(void) struct bpf_map *map; const __u32 desired_sz = sizeof(skel->bss->sum) + sysconf(_SC_PAGE_SIZE) * 2; size_t array_len, actual_sz, new_sz; + int *array; skel = test_global_map_resize__open(); if (!ASSERT_OK_PTR(skel, "test_global_map_resize__open")) @@ -58,10 +59,13 @@ static void global_map_resize_bss_subtest(void) goto teardown; /* fill the newly resized array with ones, - * skipping the first element which was previously set + * skipping the first element which was previously set; + * access through a plain pointer to avoid -Warray-bounds + * since the array was resized beyond its declared length. */ + array = skel->bss->array; for (int i = 1; i < array_len; i++) - skel->bss->array[i] = 1; + array[i] = 1; /* set global const values before loading */ skel->rodata->pid = getpid(); diff --git a/tools/testing/selftests/bpf/prog_tests/libarena.c b/tools/testing/selftests/bpf/prog_tests/libarena.c index 61ea68dce4105f..ba5a5a50f7c07d 100644 --- a/tools/testing/selftests/bpf/prog_tests/libarena.c +++ b/tools/testing/selftests/bpf/prog_tests/libarena.c @@ -15,7 +15,12 @@ static void run_libarena_test(struct libarena *skel, struct bpf_program *prog, { int ret; - if (!strstr(name, "test_buddy")) { + if (strstr(name, "test_buddy")) { + /* Buddy tests initialize the allocator directly. */ + ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_destroy)); + if (!ASSERT_OK(ret, "arena_buddy_destroy")) + return; + } else { ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_reset)); if (!ASSERT_OK(ret, "arena_buddy_reset")) return; @@ -24,7 +29,6 @@ static void run_libarena_test(struct libarena *skel, struct bpf_program *prog, ret = libarena_run_prog(bpf_program__fd(prog)); ASSERT_OK(ret, name); - } static void *run_libarena_parallel_prog(void *arg) diff --git a/tools/testing/selftests/bpf/prog_tests/libarena_asan.c b/tools/testing/selftests/bpf/prog_tests/libarena_asan.c index d59d9dd12ef2b2..f897405f701dd5 100644 --- a/tools/testing/selftests/bpf/prog_tests/libarena_asan.c +++ b/tools/testing/selftests/bpf/prog_tests/libarena_asan.c @@ -17,7 +17,12 @@ static void run_libarena_asan_test(struct libarena_asan *skel, { int ret; - if (!strstr(name, "test_buddy")) { + if (strstr(name, "test_buddy")) { + /* Buddy tests initialize the allocator directly. */ + ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_destroy)); + if (!ASSERT_OK(ret, "arena_buddy_destroy")) + return; + } else { ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_reset)); if (!ASSERT_OK(ret, "arena_buddy_reset")) return; @@ -90,4 +95,3 @@ void test_libarena_asan(void) return; } - diff --git a/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c b/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c index a970798e117353..28bc4b117f415b 100644 --- a/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c +++ b/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include "lsm_bdev.skel.h" @@ -172,7 +173,7 @@ void test_lsm_bdev(void) if (!ASSERT_OK(stat(DM_DEV_PATH, &st), "stat dm dev")) goto remove_dm; - dev_key = (__u32)st.st_rdev; + dev_key = (major(st.st_rdev) << 20) | minor(st.st_rdev); /* Look up the device in the BPF map and verify. */ err = bpf_map__lookup_elem(skel->maps.verity_devices, diff --git a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c index 6606f0ed9a9a9e..39e8a3b8b6afb9 100644 --- a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c +++ b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c @@ -410,7 +410,8 @@ close_netns: static int check_ping_ok(const char *ns1) { SYS(fail, "ip netns exec %s ping -c 1 -W1 -I veth1 %s > /dev/null", ns1, IP4_ADDR_DST); - SYS(fail, "ip netns exec %s ping6 -c 1 -W1 -I veth1 %s > /dev/null", ns1, IP6_ADDR_DST); + SYS(fail, "ip netns exec %s %s -c 1 -W1 -I veth1 %s > /dev/null", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); return 0; fail: return -1; @@ -424,7 +425,8 @@ static int check_ping_fails(const char *ns1) if (!ret) return -1; - ret = SYS_NOFAIL("ip netns exec %s ping6 -c 1 -W1 -I veth1 %s", ns1, IP6_ADDR_DST); + ret = SYS_NOFAIL("ip netns exec %s %s -c 1 -W1 -I veth1 %s", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); if (!ret) return -1; @@ -657,9 +659,10 @@ static void lwt_ip_encap_vxlan(bool ipv4_encap) skel->bss->fexit_triggered = false; if (ipv4_encap) - SYS(out, "ip netns exec %s ping -c 1 -W1 %s", ns1, IP4_ADDR_DST); + SYS(out, "ip netns exec %s ping -c 1 -W1 %s", ns1, IP4_ADDR_DST); else - SYS(out, "ip netns exec %s ping6 -c 1 -W1 %s", ns1, IP6_ADDR_DST); + SYS(out, "ip netns exec %s %s -c 1 -W1 %s", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); if (!ASSERT_TRUE(skel->bss->fexit_triggered, "fexit_triggered")) goto out; diff --git a/tools/testing/selftests/bpf/prog_tests/mptcp.c b/tools/testing/selftests/bpf/prog_tests/mptcp.c index 8fade8bdc4516c..32dfc1c511af62 100644 --- a/tools/testing/selftests/bpf/prog_tests/mptcp.c +++ b/tools/testing/selftests/bpf/prog_tests/mptcp.c @@ -264,7 +264,7 @@ static int verify_mptcpify(int server_fd, int client_fd) return err; } -static int run_mptcpify(int cgroup_fd) +static int run_mptcpify(int cgroup_fd, int type) { int server_fd, client_fd, err = 0; struct mptcpify *mptcpify_skel; @@ -280,7 +280,7 @@ static int run_mptcpify(int cgroup_fd) goto out; /* without MPTCP */ - server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0); + server_fd = start_server(AF_INET, type, NULL, 0, 0); if (!ASSERT_GE(server_fd, 0, "start_server")) { err = -EIO; goto out; @@ -317,7 +317,14 @@ static void test_mptcpify(void) if (!ASSERT_OK_PTR(netns, "netns_new")) goto fail; - ASSERT_OK(run_mptcpify(cgroup_fd), "run_mptcpify"); + ASSERT_OK(run_mptcpify(cgroup_fd, SOCK_STREAM), "run_mptcpify"); + /* userspace sets flags such as SOCK_CLOEXEC together with the type; + * the BPF prog must still upgrade the socket to MPTCP. See + * update_socket_protocol() in net/socket.c, which runs before the + * type is masked with SOCK_TYPE_MASK. + */ + ASSERT_OK(run_mptcpify(cgroup_fd, SOCK_STREAM | SOCK_CLOEXEC), + "run_mptcpify_cloexec"); fail: netns_free(netns); diff --git a/tools/testing/selftests/bpf/prog_tests/sha256.c b/tools/testing/selftests/bpf/prog_tests/sha256.c index 604a0b1423d554..5edbc6194b0712 100644 --- a/tools/testing/selftests/bpf/prog_tests/sha256.c +++ b/tools/testing/selftests/bpf/prog_tests/sha256.c @@ -25,10 +25,10 @@ void test_sha256(void) size_t i; data = malloc(MAX_LEN); - if (!ASSERT_OK_PTR(data, "malloc")) + if (!ASSERT_NEQ(data, NULL, "malloc")) goto out; digests = malloc((MAX_LEN + 1) * SHA256_DIGEST_LENGTH); - if (!ASSERT_OK_PTR(digests, "malloc")) + if (!ASSERT_NEQ(digests, NULL, "malloc")) goto out; /* Generate MAX_LEN bytes of "random" data deterministically. */ diff --git a/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c b/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c index 955a37751b52d5..c944136252c6d6 100644 --- a/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c +++ b/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c @@ -762,7 +762,7 @@ static void test_uprobe_error(void) long err = syscall(__NR_uprobe); ASSERT_EQ(err, -1, "error"); - ASSERT_EQ(errno, ENXIO, "errno"); + ASSERT_EQ(errno, EPROTO, "errno"); } static void __test_uprobe_syscall(void) diff --git a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h index d8dacef37c1639..c4b4388545650f 100644 --- a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h +++ b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h @@ -8,6 +8,9 @@ #define AF_INET 2 #define AF_INET6 10 +/* include/linux/net.h */ +#define SOCK_TYPE_MASK 0xf + #define SOL_SOCKET 1 #define SO_REUSEADDR 2 #define SO_SNDBUF 7 diff --git a/tools/testing/selftests/bpf/progs/mptcpify.c b/tools/testing/selftests/bpf/progs/mptcpify.c index cbdc730c3a4716..e3f8cb54dbe97e 100644 --- a/tools/testing/selftests/bpf/progs/mptcpify.c +++ b/tools/testing/selftests/bpf/progs/mptcpify.c @@ -15,7 +15,7 @@ int BPF_PROG(mptcpify, int family, int type, int protocol) return protocol; if ((family == AF_INET || family == AF_INET6) && - type == SOCK_STREAM && + (type & SOCK_TYPE_MASK) == SOCK_STREAM && (!protocol || protocol == IPPROTO_TCP)) { return IPPROTO_MPTCP; } diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index 3ce32d134e2cce..07807757b518d6 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -807,7 +807,7 @@ static void verify_stderr(int prog_fd, struct expected_msgs *msgs) return; buf = malloc(TEST_LOADER_LOG_BUF_SZ); - if (!ASSERT_OK_PTR(buf, "malloc")) + if (!ASSERT_NEQ(buf, NULL, "malloc")) return; ret = bpf_prog_stream_read(prog_fd, 2, buf, TEST_LOADER_LOG_BUF_SZ - 1, diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index 7ba82974ee784d..8ffababe0084bc 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c @@ -573,18 +573,19 @@ bool test__start_subtest_with_desc(const char *subtest_name, const char *subtest struct subtest_state *subtest_state; const char *subtest_display_name; size_t sub_state_size = sizeof(*subtest_state); + void *tmp; if (env.subtest_state) test__end_subtest(); state->subtest_num++; - state->subtest_states = - realloc(state->subtest_states, - state->subtest_num * sub_state_size); - if (!state->subtest_states) { + tmp = realloc(state->subtest_states, state->subtest_num * sub_state_size); + if (!tmp) { + state->subtest_num--; fprintf(stderr, "Not enough memory to allocate subtest result\n"); return false; } + state->subtest_states = tmp; subtest_state = &state->subtest_states[state->subtest_num - 1]; @@ -730,11 +731,14 @@ int compare_map_keys(int map1_fd, int map2_fd) int compare_stack_ips(int smap_fd, int amap_fd, int stack_trace_len) { __u32 key, next_key, *cur_key_p, *next_key_p; - char *val_buf1, *val_buf2; - int i, err = 0; + char *val_buf1 = NULL, *val_buf2 = NULL; + int i, err = -ENOMEM; val_buf1 = malloc(stack_trace_len); val_buf2 = malloc(stack_trace_len); + if (!val_buf1 || !val_buf2) + goto out; + err = 0; cur_key_p = NULL; next_key_p = &key; while (bpf_map_get_next_key(smap_fd, cur_key_p, next_key_p) == 0) { @@ -1514,6 +1518,10 @@ static int dispatch_thread_send_subtests(int sock_fd, struct test_state *state) int subtest_num = state->subtest_num; state->subtest_states = malloc(subtest_num * sizeof(*subtest_state)); + if (!state->subtest_states) { + state->subtest_num = 0; + return -ENOMEM; + } for (int i = 0; i < subtest_num; i++) { subtest_state = &state->subtest_states[i]; @@ -1741,7 +1749,7 @@ static void server_main(void) data[i].worker_id = i; data[i].sock_fd = env.worker_socks[i]; rc = pthread_create(&dispatcher_threads[i], NULL, dispatch_thread, &data[i]); - if (rc < 0) { + if (rc) { perror("Failed to launch dispatcher thread"); exit(EXIT_ERR_SETUP_INFRA); } diff --git a/tools/testing/selftests/bpf/test_sockmap.c b/tools/testing/selftests/bpf/test_sockmap.c index ac814eb63edb62..aaf2050e884509 100644 --- a/tools/testing/selftests/bpf/test_sockmap.c +++ b/tools/testing/selftests/bpf/test_sockmap.c @@ -435,7 +435,8 @@ static int msg_alloc_iov(struct msghdr *msg, return 0; unwind_iov: for (i--; i >= 0 ; i--) - free(msg->msg_iov[i].iov_base); + free(iov[i].iov_base); + free(iov); return -ENOMEM; } diff --git a/tools/testing/selftests/bpf/veristat.c b/tools/testing/selftests/bpf/veristat.c index a7db6f04f7e145..96f7e43ff0a8d7 100644 --- a/tools/testing/selftests/bpf/veristat.c +++ b/tools/testing/selftests/bpf/veristat.c @@ -514,6 +514,40 @@ cleanup: return err == 0; } +/* Exact filter match */ +static bool name_filter_matches(struct filter *f, const char *filename, const char *prog_name) +{ + if (f->any_glob) + return glob_matches(filename, f->any_glob) || + (prog_name && glob_matches(prog_name, f->any_glob)); + if (f->file_glob && f->prog_glob) + return prog_name && + glob_matches(filename, f->file_glob) && + glob_matches(prog_name, f->prog_glob); + if (f->file_glob) + return glob_matches(filename, f->file_glob); + if (f->prog_glob) + return prog_name && glob_matches(prog_name, f->prog_glob); + return false; +} + +/* Check if the filter does not outright reject the file name */ +static bool name_filter_may_match(struct filter *f, const char *filename) +{ + if (f->file_glob) + return glob_matches(filename, f->file_glob); + /* + * If we don't know program name yet, any_glob filter + * has to assume that current BPF object file might be + * relevant; we'll check again later on after opening + * BPF object file, at which point program name will + * be known finally. + */ + if (f->any_glob || f->prog_glob) + return true; + return false; +} + static bool should_process_file_prog(const char *filename, const char *prog_name) { struct filter *f; @@ -521,16 +555,7 @@ static bool should_process_file_prog(const char *filename, const char *prog_name for (i = 0; i < env.deny_filter_cnt; i++) { f = &env.deny_filters[i]; - if (f->kind != FILTER_NAME) - continue; - - if (f->any_glob && glob_matches(filename, f->any_glob)) - return false; - if (f->any_glob && prog_name && glob_matches(prog_name, f->any_glob)) - return false; - if (f->file_glob && glob_matches(filename, f->file_glob)) - return false; - if (f->prog_glob && prog_name && glob_matches(prog_name, f->prog_glob)) + if (f->kind == FILTER_NAME && name_filter_matches(f, filename, prog_name)) return false; } @@ -540,24 +565,15 @@ static bool should_process_file_prog(const char *filename, const char *prog_name continue; allow_cnt++; - if (f->any_glob) { - if (glob_matches(filename, f->any_glob)) - return true; - /* If we don't know program name yet, any_glob filter - * has to assume that current BPF object file might be - * relevant; we'll check again later on after opening - * BPF object file, at which point program name will - * be known finally. - */ - if (!prog_name || glob_matches(prog_name, f->any_glob)) - return true; - } else { - if (f->file_glob && !glob_matches(filename, f->file_glob)) - continue; - if (f->prog_glob && prog_name && !glob_matches(prog_name, f->prog_glob)) - continue; + if (prog_name && name_filter_matches(f, filename, prog_name)) + return true; + /* + * If there is no prog_name and the file name is not blocked by + * the filter, allow to open the file. Afterwards there would be + * a second refining query with prog_name set. + */ + if (!prog_name && name_filter_may_match(f, filename)) return true; - } } /* if there are no file/prog name allow filters, allow all progs, @@ -703,6 +719,12 @@ static int append_filter(struct filter **filters, int *cnt, const char *str) } } + if ((!f->any_glob && !f->file_glob && !f->prog_glob) || + (f->any_glob && strcmp(f->any_glob, "") == 0)) { + fprintf(stderr, "Invalid filter: '%s'\n", str); + return -EINVAL; + } + *cnt += 1; return 0; } diff --git a/tools/testing/selftests/bpf/vmtest.sh b/tools/testing/selftests/bpf/vmtest.sh index 9ca8022853933a..6a3d026d76bd6b 100755 --- a/tools/testing/selftests/bpf/vmtest.sh +++ b/tools/testing/selftests/bpf/vmtest.sh @@ -428,8 +428,17 @@ main() if [[ $# -eq 0 && "${debug_shell}" == "no" ]]; then echo "No command specified, will run ${DEFAULT_COMMAND} in the vm" - else - command="$@" + elif [[ $# -gt 0 ]]; then + # Quote each argument so the command survives into the guest: the + # host expands ${command} into the generated init script, which + # the guest bash then parses as shell source. Without the %q + # escapes an argument with a space or a glob (e.g. -a 'verifier_*') + # is re-split and expanded against /root/bpf there. + # + # Skip this when there is no command: printf '%q ' would still + # apply the format once and emit '', which is not the empty + # command that -s (debug shell) expects. + command=$(printf '%q ' "$@") fi local kconfig_file="${OUTPUT_DIR}/latest.config" diff --git a/tools/testing/selftests/cgroup/test_cpuset_prs.sh b/tools/testing/selftests/cgroup/test_cpuset_prs.sh index 0d41aa0d343d9f..b2e60671273ec1 100755 --- a/tools/testing/selftests/cgroup/test_cpuset_prs.sh +++ b/tools/testing/selftests/cgroup/test_cpuset_prs.sh @@ -20,7 +20,7 @@ skip_test() { WAIT_INOTIFY=$(cd $(dirname $0); pwd)/wait_inotify # Find cgroup v2 mount point -CGROUP2=$(mount -t cgroup2 | head -1 | awk -e '{print $3}') +CGROUP2=$(mount -t cgroup2 | head -1 | awk '{print $3}') [[ -n "$CGROUP2" ]] || skip_test "Cgroup v2 mount point not found!" SUBPARTS_CPUS=$CGROUP2/.__DEBUG__.cpuset.cpus.subpartitions CPULIST=$(cat $CGROUP2/cpuset.cpus.effective) @@ -495,13 +495,20 @@ REMOTE_TEST_MATRIX=( # Narrowing cpuset.cpus to previously sibling-excluded CPUs should # not return CPUs that were never actually owned. " C1-4:P1 . C1-2:P1 C1-3:P2 . . \ - . . . C3 . . p1:4|c11:1-2|c12:3 \ + . . . C3 . . p1:4|c11:1-2|c12:3 \ p1:P1|c11:P1|c12:P2 3" # Expanding cpuset.cpus to include a previously sibling-excluded CPU # after the sibling has become a member should correctly request it. " C1-4:P1 . C1-2:P1 C1-3:P2 . . \ - . . P0 C2-3 . . p1:1,4|c11:1|c12:2-3 \ + . . P0 C2-3 . . p1:1,4|c11:1|c12:2-3 \ p1:P1|c11:P0|c12:P2 2-3" + # Cpusets with empty cpuset.cpus should inherit parent's effective_cpus + " C1-4:P1 C5-6 C1-2 . C5 . \ + . P1 P1 . . . p1:3-4|p2:5-6|c11:1-2|c12:3-4|c21:5|c22:5-6 \ + p1:P1|p2:P1|c11:P1" + " C1-4:P1 C5-6 C1-2 . C5 . \ + . P1 P1 . O5=0 . p1:3-4|p2:6|c11:1-2|c12:3-4|c21:6|c22:6 \ + p1:P1|p2:P1|c11:P1" ) # @@ -513,6 +520,7 @@ write_cpu_online() CPU=${1%=*} VAL=${1#*=} CPUFILE=//sys/devices/system/cpu/cpu${CPU}/online + echo $VAL > $CPUFILE || return 1 if [[ $VAL -eq 0 ]] then OFFLINE_CPUS="$OFFLINE_CPUS $CPU" @@ -522,7 +530,6 @@ write_cpu_online() sort | uniq -u) } fi - echo $VAL > $CPUFILE pause 0.05 } @@ -590,7 +597,8 @@ set_ctrl_state() eval $COMM $REDIRECT ;; O*) VAL=${CMD#?} - write_cpu_online $VAL + COMM="write_cpu_online $VAL" + eval $COMM $REDIRECT ;; T*) COMM="echo 0 > $TFILE" eval $COMM $REDIRECT diff --git a/tools/testing/selftests/cgroup/test_cpuset_v1_hp.sh b/tools/testing/selftests/cgroup/test_cpuset_v1_hp.sh index 7406c24be1ac99..da97f1643f9a35 100755 --- a/tools/testing/selftests/cgroup/test_cpuset_v1_hp.sh +++ b/tools/testing/selftests/cgroup/test_cpuset_v1_hp.sh @@ -14,7 +14,7 @@ skip_test() { [[ $(id -u) -eq 0 ]] || skip_test "Test must be run as root!" # Find cpuset v1 mount point -CPUSET=$(mount -t cgroup | grep cpuset | head -1 | awk -e '{print $3}') +CPUSET=$(mount -t cgroup | grep cpuset | head -1 | awk '{print $3}') [[ -n "$CPUSET" ]] || skip_test "cpuset v1 mount point not found!" # diff --git a/tools/testing/selftests/drivers/net/config b/tools/testing/selftests/drivers/net/config index 2070e890e064e5..b6989c7d3d9d1f 100644 --- a/tools/testing/selftests/drivers/net/config +++ b/tools/testing/selftests/drivers/net/config @@ -4,8 +4,12 @@ CONFIG_DEBUG_INFO_BTF_MODULES=n CONFIG_INET_PSP=y CONFIG_IPV6=y CONFIG_MACSEC=m +CONFIG_NET_ACT_SKBEDIT=m CONFIG_NET_CLS_ACT=y CONFIG_NET_CLS_BPF=y +CONFIG_NET_CLS_FLOWER=m +CONFIG_NET_CLS_FW=m +CONFIG_NET_CLS_MATCHALL=m CONFIG_NETCONSOLE=m CONFIG_NETCONSOLE_DYNAMIC=y CONFIG_NETCONSOLE_EXTENDED_LOG=y @@ -14,6 +18,7 @@ CONFIG_NETKIT=y CONFIG_NET_SCH_ETF=m CONFIG_NET_SCH_FQ=m CONFIG_NET_SCH_INGRESS=y +CONFIG_NET_SCH_PRIO=m CONFIG_PPP=y CONFIG_PPPOE=y CONFIG_VLAN_8021Q=m diff --git a/tools/testing/selftests/drivers/net/lib/py/env.py b/tools/testing/selftests/drivers/net/lib/py/env.py index e4ab99b905b168..0156a9572000f6 100644 --- a/tools/testing/selftests/drivers/net/lib/py/env.py +++ b/tools/testing/selftests/drivers/net/lib/py/env.py @@ -114,10 +114,11 @@ class NetDrvEpEnv(NetDrvEnvBase): nsim_v4_pfx = "192.0.2." nsim_v6_pfx = "2001:db8::" - def __init__(self, src_path, nsim_test=None): + def __init__(self, src_path, nsim_test=None, queue_count=None): super().__init__(src_path) self._stats_settle_time = None + self._queue_count = queue_count # Things we try to destroy self.remote = None @@ -179,9 +180,13 @@ class NetDrvEpEnv(NetDrvEnvBase): self._required_cmd = {} def create_local(self): + nsim_kwargs = {} + if self._queue_count: + nsim_kwargs["queue_count"] = self._queue_count + self._netns = NetNS() - self._ns = NetdevSimDev() - self._ns_peer = NetdevSimDev(ns=self._netns) + self._ns = NetdevSimDev(**nsim_kwargs) + self._ns_peer = NetdevSimDev(ns=self._netns, **nsim_kwargs) with open("/proc/self/ns/net") as nsfd0, \ open("/var/run/netns/" + self._netns.name) as nsfd1: diff --git a/tools/testing/selftests/drivers/net/ring_reconfig.py b/tools/testing/selftests/drivers/net/ring_reconfig.py index f9530a8b08567f..2bc329b771340b 100755 --- a/tools/testing/selftests/drivers/net/ring_reconfig.py +++ b/tools/testing/selftests/drivers/net/ring_reconfig.py @@ -5,10 +5,25 @@ Test channel and ring size configuration via ethtool (-L / -G). """ +import socket +import struct +import time + from lib.py import ksft_run, ksft_exit, ksft_pr from lib.py import ksft_eq +from lib.py import KsftSkipEx, KsftXfailEx from lib.py import NetDrvEpEnv, EthtoolFamily, GenerateTraffic -from lib.py import defer, NlError +from lib.py import cmd, defer, rand_port, tc, NlError + +# Added in Python 3.13; fallback to 61 for x86/ARM/MIPS +SO_TXTIME = getattr(socket, "SO_TXTIME", 61) + +# Not always exported by the socket module; asm-generic value (x86/ARM/MIPS). +SO_SNDBUFFORCE = getattr(socket, "SO_SNDBUFFORCE", 32) + +# TX ring size the test shrinks to so the ring fills quickly. +MIN_TX_RING = 32 +MAX_TX_RING = 1024 def channels(cfg) -> None: @@ -151,14 +166,248 @@ def ringparam(cfg) -> None: GenerateTraffic(cfg).wait_pkts_and_stop(10000) +def _write_file(path, val): + """Write val to a file.""" + with open(path, "w", encoding="utf-8") as fp: + fp.write(str(val)) + + +def _write_sysfs(path, val): + """Write val to a sysfs file, restoring the original value on exit.""" + with open(path, "r", encoding="utf-8") as fp: + orig_val = fp.read().strip() + if str(val) == orig_val: + return + _write_file(path, val) + defer(_write_file, path, orig_val) + + +def _get_qdisc_backlog(cfg, mq_handle, queue): + """Return the qdisc backlog (bytes) for the given TX queue's leaf.""" + target_parent = f"{mq_handle}{queue + 1:x}" + for q in tc(f"-s qdisc show dev {cfg.ifname}", json=True): + if q.get("parent", "") == target_parent: + return q.get("backlog") or 0 + return 0 + + +def _setup_fq_qdisc(cfg, port, target_queue, other_queue, flow_limit): + """Put an fq qdisc on target_queue's leaf and return the mq handle in use. + + We must not disturb the device's existing TX/RX qdisc policy. On a real + NIC the root mq already has an addressable handle, so we leave the root + and every other queue alone and only swap this one leaf, restoring its + original qdisc afterwards. + + @flow_limit raises fq's per-flow packet limit (default 100) so a single + flow can back up more packets than the Tx ring holds and thus overflow it. + """ + qdiscs = tc(f"qdisc show dev {cfg.ifname}", json=True) + root = next((q for q in qdiscs if q.get("root")), None) + + if root and root["kind"] == "mq" and root["handle"] != "0:": + # Addressable mq (previously-configured): touch only the target queue's + # leaf and restore its original qdisc afterwards. + mq_handle = root["handle"] + parent = f"{mq_handle}{target_queue + 1:x}" + orig = next((q for q in qdiscs if q.get("parent") == parent), None) + orig_kind = orig["kind"] if orig else \ + cmd("sysctl -n net.core.default_qdisc").stdout.strip() + defer(tc, f"qdisc replace dev {cfg.ifname} parent {parent} {orig_kind}") + elif root is None or root["kind"] in ("mq", "noqueue"): + # The auto-attached root mq has handle 0: on any device (real or sim), + # which the kernel rejects as a qdisc parent. A 0: handle means the mq + # is the untouched kernel default - no custom child qdiscs can hang off + # an unaddressable parent - so installing a real handle and restoring + # the default mq on exit preserves the device's effective policy. + mq_handle = "1:" + tc(f"qdisc replace dev {cfg.ifname} root handle {mq_handle} mq") + defer(tc, f"qdisc replace dev {cfg.ifname} root mq") + parent = f"{mq_handle}{target_queue + 1:x}" + else: + raise KsftSkipEx(f"root qdisc '{root['kind']}' is not mq; " + "refusing to disturb existing qdisc policy") + + try: + tc(f"qdisc replace dev {cfg.ifname} parent {parent} fq " + f"flow_limit {flow_limit} limit {flow_limit * 2}") + except Exception as exc: + raise KsftSkipEx( + f"fq not available (CONFIG_NET_SCH_FQ): {exc}") from exc + + qdisc_j = tc(f"qdisc show dev {cfg.ifname}", json=True) + has_clsact = any(q['kind'] == 'clsact' for q in qdisc_j) + if not has_clsact: + tc(f"qdisc add dev {cfg.ifname} clsact") + defer(tc, f"qdisc del dev {cfg.ifname} clsact") + + proto = "ipv6" if int(cfg.addr_ipver) == 6 else "ip" + try: + tc(f"filter add dev {cfg.ifname} egress protocol {proto} " + f"pref 1 flower ip_proto udp dst_port {port} " + f"action skbedit queue_mapping {target_queue}") + except Exception as exc: + raise KsftSkipEx("tc flower/act_skbedit not available") from exc + defer(tc, f"filter del dev {cfg.ifname} egress pref 1") + + tc(f"filter add dev {cfg.ifname} egress pref 101 " + f"matchall action skbedit queue_mapping {other_queue}") + defer(tc, f"filter del dev {cfg.ifname} egress pref 101") + + return mq_handle + + +def _create_sotxtime_socket(cfg, sndbuf): + """Create a UDP socket with SO_TXTIME enabled, bound to the test device.""" + sock = socket.socket(socket.AF_INET6 if cfg.addr_ipver == "6" + else socket.AF_INET, socket.SOCK_DGRAM) + try: + sock.setsockopt(socket.SOL_SOCKET, SO_TXTIME, struct.pack("Ii", 1, 0)) + except OSError as exc: + sock.close() + raise KsftSkipEx("SO_TXTIME not supported") from exc + sock.setsockopt(socket.SOL_SOCKET, socket.SO_BINDTODEVICE, + cfg.ifname.encode()) + # Deferred completions keep every in-flight skb charged to the socket, so + # size the send buffer to hold the whole burst. SO_SNDBUFFORCE bypasses + # net.core.wmem_max (the test runs as root). + try: + sock.setsockopt(socket.SOL_SOCKET, SO_SNDBUFFORCE, sndbuf) + except OSError: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, sndbuf) + return sock + + +def _send_sotxtime_burst(cfg, sock, port, count, delay_ns, pkt_size): + """Send count UDP packets scheduled delay_ns ahead using SO_TXTIME.""" + payload = b'\x00' * pkt_size + txtime_ns = time.clock_gettime_ns(time.CLOCK_MONOTONIC) + delay_ns + + ancdata = [(socket.SOL_SOCKET, SO_TXTIME, struct.pack("Q", txtime_ns))] + if int(cfg.addr_ipver) == 6: + dest = (cfg.remote_addr, port, 0, 0) + else: + dest = (cfg.remote_addr, port) + for _ in range(count): + sock.sendmsg([payload], ancdata, 0, dest) + + +def _set_small_tx_ring(cfg, ehdr): + """Set the Tx ring to the smallest size the driver accepts. + + Start at 32 so the ring fills quickly, then grow exponentially (64, + 128, 256, ...) up to 1024. Some drivers enforce a minimum well above 32 + (e.g. bnxt needs a large ring for software UDP segmentation), so raise + the lower bound until the driver accepts it, giving up past 1024. + """ + size = MIN_TX_RING + while size <= MAX_TX_RING: + try: + cfg.eth.rings_set(ehdr | {'tx': size}) + return size + except NlError: + size = size * 2 + continue + raise KsftSkipEx("driver rejects all tx ring sizes up to 1024") + + +def reconfig_tx_stall(cfg) -> None: + """Test that qdisc backlog drains after ring reconfiguration.""" + target_queue = 1 + other_queue = 0 + + ehdr = {'header': {'dev-index': cfg.ifindex}} + chans = cfg.eth.channels_get(ehdr) + + if "combined-max" not in chans: + raise KsftSkipEx("device does not support combined channels") + if chans.get("combined-max", 0) < 2: + raise KsftSkipEx("device does not support 2+ combined channels") + if chans["combined-count"] < 2: + defer(cfg.eth.channels_set, + ehdr | {"combined-count": chans["combined-count"]}) + cfg.eth.channels_set(ehdr | {"combined-count": 2}) + + rings = cfg.eth.rings_get(ehdr) + if 'rx' not in rings or 'tx' not in rings: + raise KsftSkipEx("device does not expose rx/tx ring params") + tx_cur = rings['tx'] + if tx_cur <= MIN_TX_RING: + raise KsftSkipEx("tx ring size already at minimum") + defer(cfg.eth.rings_set, ehdr | {'tx': tx_cur}) + + # Use the smallest Tx ring the driver accepts (32, growing to 1024). + tx_ring = _set_small_tx_ring(cfg, ehdr) + + # Slow completions so the ring stays full after FQ releases packets + napi_defer = f"/sys/class/net/{cfg.ifname}/napi_defer_hard_irqs" + gro_timeout = f"/sys/class/net/{cfg.ifname}/gro_flush_timeout" + _write_sysfs(napi_defer, 100) + _write_sysfs(gro_timeout, 1000000000) + + port = rand_port() + # A single flow must overflow the ring, so send twice the ring depth and + # let fq hold that many packets for the flow. + pkt_count = tx_ring * 2 + mq_handle = _setup_fq_qdisc(cfg, port, target_queue, other_queue, + tx_ring * 2) + + # Size each packet to one MTU (less L3/L4 headers to avoid fragmentation). + pkt_size = cfg.dev['mtu'] - (48 if int(cfg.addr_ipver) == 6 else 28) + + # Each queued skb charges the socket its truesize (~2x the payload), so + # budget the send buffer for the whole in-flight burst. + sock = _create_sotxtime_socket(cfg, pkt_count * pkt_size * 2) + defer(sock.close) + + for delay_ms in [100, 200, 500]: + _send_sotxtime_burst(cfg, sock, port, pkt_count, + delay_ms * 1_000_000, pkt_size) + ksft_pr(f"Sent {pkt_count} SO_TXTIME packets (+{delay_ms}ms)") + time.sleep(delay_ms / 1000 + 0.3) + + backlog = _get_qdisc_backlog(cfg, mq_handle, target_queue) + if backlog: + break + else: + # A device that completes Tx synchronously (e.g. a software/virtual + # driver like netdevsim) never keeps the ring full long enough for a + # backlog to form, so the wake-vs-start behavior can't be exercised. + # Treat that as an expected failure rather than a hard failure. + raise KsftXfailEx("could not build qdisc backlog") + + ksft_pr(f"Backlog before reconfig: {backlog} bytes") + + # Trigger ring reconfig — driver should call wake, not just start. + # Grow back to the original size so the driver actually switches channels + # (setting the current size is a no-op the driver short-circuits). + cfg.eth.rings_set(ehdr | {'tx': tx_cur}) + + # Let completions proceed normally + _write_sysfs(napi_defer, 0) + _write_sysfs(gro_timeout, 0) + + # Poll for backlog to drain + for _ in range(100): + backlog = _get_qdisc_backlog(cfg, mq_handle, target_queue) + if not backlog: + break + time.sleep(0.1) + + ksft_eq(0, backlog, + comment=f"qdisc backlog stuck on queue {target_queue} " + f"after ring reconfig") + + def main() -> None: """ Ksft boiler plate main """ - with NetDrvEpEnv(__file__) as cfg: + with NetDrvEpEnv(__file__, queue_count=2) as cfg: cfg.eth = EthtoolFamily() ksft_run([channels, - ringparam], + ringparam, + reconfig_tx_stall], args=(cfg, )) ksft_exit() diff --git a/tools/testing/selftests/drivers/net/so_txtime.py b/tools/testing/selftests/drivers/net/so_txtime.py index adf6c848d6d80f..a097fae0b335a3 100755 --- a/tools/testing/selftests/drivers/net/so_txtime.py +++ b/tools/testing/selftests/drivers/net/so_txtime.py @@ -12,6 +12,7 @@ import time from lib.py import ksft_exit, ksft_run, ksft_variants from lib.py import KsftNamedVariant, KsftSkipEx from lib.py import NetDrvEpEnv, bkg, cmd, defer, tc +from lib.py import CmdExitFailure def test_so_txtime(cfg, clockid, ipver, args_tx, args_rx, expect_success): @@ -27,7 +28,7 @@ def test_so_txtime(cfg, clockid, ipver, args_tx, args_rx, expect_success): cmd_addr = f"-S {cfg.addr_v[ipver]} -D {cfg.remote_addr_v[ipver]}" cmd_args = f"-{ipver} -c {clockid} -t {tstart} {cmd_addr}" cmd_rx = f"{cfg.bin_remote} {cmd_args} {args_rx} -r" - cmd_tx = f"{cfg.bin_local} {cmd_args} {args_tx}" + cmd_tx = f"{cfg.bin_local} -m 100 {cmd_args} {args_tx}" expect_fail = not expect_success if slow_machine: @@ -45,7 +46,11 @@ def _qdisc_setup(ifname, qdisc, optargs=""): """ orig = tc(f"qdisc show dev {ifname} root", json=True)[0].get("kind", None) defer(tc, f"qdisc replace dev {ifname} root {orig}") - tc(f"qdisc replace dev {ifname} root {qdisc} {optargs}") + try: + tc(f"qdisc del dev {ifname} root") + except CmdExitFailure: + pass + tc(f"qdisc replace dev {ifname} root handle 1: {qdisc} {optargs}") def _test_variants_fq(): @@ -96,11 +101,21 @@ def _test_variants_etf(): def test_so_txtime_etf(cfg, ipver, args_tx, args_rx, expect_fail): """Run all variants of etf tests.""" cfg.require_ipver(ipver) + + # root qdisc for background traffic (e.g., bkg()) + _qdisc_setup(cfg.ifname, "prio") + + # leaf ETF qdisc only for intended packets try: - _qdisc_setup(cfg.ifname, "etf", "clockid CLOCK_TAI delta 400000") + etf_args = "clockid CLOCK_TAI delta 400000" + tc(f"qdisc add dev {cfg.ifname} parent 1:1 handle 10: etf {etf_args}") except Exception as e: raise KsftSkipEx("tc does not support qdisc etf. skipping") from e + # redirect mark 100 to leaf + filter_args = "protocol all handle 100 fw flowid 1:1" + tc(f"filter add dev {cfg.ifname} parent 1: {filter_args}") + test_so_txtime(cfg, "tai", ipver, args_tx, args_rx, expect_fail) diff --git a/tools/testing/selftests/kselftest_harness.h b/tools/testing/selftests/kselftest_harness.h index 261e4df94d9d5b..29a19bc870846f 100644 --- a/tools/testing/selftests/kselftest_harness.h +++ b/tools/testing/selftests/kselftest_harness.h @@ -467,7 +467,7 @@ static inline void __kselftest_memset_safe(void *s, int c, size_t n) !__atomic_test_and_set(_metadata->no_teardown, __ATOMIC_RELAXED)) \ fixture_name##_teardown(_metadata, self, variant); \ } \ - static struct __test_metadata *_##fixture_name##_##test_name##_object; \ + static struct __test_metadata *_##fixture_name##_##test_name##_object __maybe_unused; \ static void __attribute__((constructor(KSELFTEST_PRIO_TEST))) \ _register_##fixture_name##_##test_name(void) \ { \ diff --git a/tools/testing/selftests/lsm/common.c b/tools/testing/selftests/lsm/common.c index 9ad258912646c8..927dce4f04cb2c 100644 --- a/tools/testing/selftests/lsm/common.c +++ b/tools/testing/selftests/lsm/common.c @@ -76,7 +76,7 @@ int attr_lsm_count(void) return 0; if (read_sysfs_lsms(names, sysconf(_SC_PAGESIZE))) - return 0; + goto out; if (strstr(names, "selinux")) count++; @@ -85,5 +85,7 @@ int attr_lsm_count(void) if (strstr(names, "apparmor")) count++; +out: + free(names); return count; } diff --git a/tools/testing/selftests/mm/cow.c b/tools/testing/selftests/mm/cow.c index 0c627ea89ff7b2..c1b8920e293424 100644 --- a/tools/testing/selftests/mm/cow.c +++ b/tools/testing/selftests/mm/cow.c @@ -1718,8 +1718,13 @@ static void run_with_tmpfile(non_anon_test_fn fn, const char *desc) /* File consists of a single page filled with zeroes. */ if (fallocate(fd, 0, 0, pagesize)) { - ksft_perror("fallocate() failed"); - log_test_result(KSFT_FAIL); + if (errno == EOPNOTSUPP) { + ksft_print_msg("fallocate() not supported by filesystem\n"); + log_test_result(KSFT_SKIP); + } else { + ksft_perror("fallocate() failed"); + log_test_result(KSFT_FAIL); + } goto close; } diff --git a/tools/testing/selftests/mm/hmm-tests.c b/tools/testing/selftests/mm/hmm-tests.c index 2f2b9879d10051..6fccbdab02ee81 100644 --- a/tools/testing/selftests/mm/hmm-tests.c +++ b/tools/testing/selftests/mm/hmm-tests.c @@ -2829,8 +2829,11 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz buffer->ptr = mmap(NULL, buffer_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (buffer->ptr == MAP_FAILED) - return -1; + if (buffer->ptr == MAP_FAILED) { + buffer->ptr = NULL; + ret = -1; + goto cleanup; + } /* Apply THP hint if requested */ if (use_thp) @@ -2839,7 +2842,7 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz ret = madvise(buffer->ptr, buffer_size, MADV_NOHUGEPAGE); if (ret) - return ret; + goto cleanup; /* Initialize memory to make sure pages are allocated */ ptr = (int *)buffer->ptr; @@ -2849,11 +2852,11 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz /* Warmup iteration */ ret = hmm_migrate_sys_to_dev(fd, buffer, npages); if (ret) - return ret; + goto cleanup; ret = hmm_migrate_dev_to_sys(fd, buffer, npages); if (ret) - return ret; + goto cleanup; /* Benchmark iterations */ for (i = 0; i < iterations; i++) { @@ -2862,7 +2865,7 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz ret = hmm_migrate_sys_to_dev(fd, buffer, npages); if (ret) - return ret; + goto cleanup; end = get_time_ms(); s2d_total += (end - start); @@ -2872,7 +2875,7 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz ret = hmm_migrate_dev_to_sys(fd, buffer, npages); if (ret) - return ret; + goto cleanup; end = get_time_ms(); d2s_total += (end - start); @@ -2886,9 +2889,9 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz results->throughput_d2s = (buffer_size / (1024.0 * 1024.0 * 1024.0)) / (results->dev_to_sys_time / 1000.0); - /* Cleanup */ +cleanup: hmm_buffer_free(buffer); - return 0; + return ret; } /* diff --git a/tools/testing/selftests/mm/hugepage_settings.c b/tools/testing/selftests/mm/hugepage_settings.c index 2eab2110ac6a46..d7917dce3abac7 100644 --- a/tools/testing/selftests/mm/hugepage_settings.c +++ b/tools/testing/selftests/mm/hugepage_settings.c @@ -437,7 +437,7 @@ void hugetlb_set_nr_pages(unsigned long size, unsigned long nr) hugetlb_sysfs_path(path, sizeof(path), size, "nr_hugepages"); - write_num(path, nr); + write_num_ignore_einval(path, nr); } unsigned long hugetlb_free_pages(unsigned long size) diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index 10e8dedcb087dc..506310d4b4d5cb 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -138,8 +138,8 @@ static void get_finfo(const char *dir) major(path_stat.st_dev), minor(path_stat.st_dev)) >= sizeof(path)) ksft_exit_fail_msg("%s: Pathname is too long\n", __func__); - if (read_file(path, buf, sizeof(buf)) < 0) - ksft_exit_fail_perror("read_file(read_num)"); + if (!read_file(path, buf, sizeof(buf))) + ksft_exit_fail_perror("read_file(uevent)"); if (strstr(buf, "DEVTYPE=disk")) { /* Found it */ if (snprintf(finfo.dev_queue_read_ahead_path, diff --git a/tools/testing/selftests/mm/ksm_tests.c b/tools/testing/selftests/mm/ksm_tests.c index a050f4840cfa3a..5fd7792a0d4794 100644 --- a/tools/testing/selftests/mm/ksm_tests.c +++ b/tools/testing/selftests/mm/ksm_tests.c @@ -288,8 +288,8 @@ static bool assert_ksm_pages_count(long dupl_page_count) static int ksm_save_def(struct ksm_sysfs *ksm_sysfs) { if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) || - numa_available() ? 0 : - ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) || + (numa_available() ? 0 : + ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes)) || ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) || ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) || ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) || @@ -304,8 +304,8 @@ static int ksm_save_def(struct ksm_sysfs *ksm_sysfs) static int ksm_restore(struct ksm_sysfs *ksm_sysfs) { if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) || - numa_available() ? 0 : - ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) || + (numa_available() ? 0 : + ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes)) || ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) || ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) || ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) || @@ -440,9 +440,9 @@ static int get_next_mem_node(int node) mem_node = i % (max_node + 1); node_size = numa_node_size(mem_node, NULL); if (node_size > 0) - break; + return mem_node; } - return mem_node; + return -ENODEV; } static int get_first_mem_node(void) @@ -455,8 +455,8 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo { void *numa1_map_ptr, *numa2_map_ptr; struct timespec start_time; + int first_node, second_node; int page_count = 2; - int first_node; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { ksft_perror("clock_gettime"); @@ -467,17 +467,19 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo ksft_print_msg("NUMA support not enabled\n"); return KSFT_SKIP; } - if (numa_num_configured_nodes() <= 1) { - ksft_print_msg("At least 2 NUMA nodes must be available\n"); + first_node = get_first_mem_node(); + second_node = get_next_mem_node(first_node); + + if (second_node < 0) { + ksft_print_msg("At least 2 NUMA nodes with memory must be available\n"); return KSFT_SKIP; } if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes)) return KSFT_FAIL; /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */ - first_node = get_first_mem_node(); numa1_map_ptr = numa_alloc_onnode(page_size, first_node); - numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node)); + numa2_map_ptr = numa_alloc_onnode(page_size, second_node); if (!numa1_map_ptr || !numa2_map_ptr) { ksft_perror("numa_alloc_onnode"); return KSFT_FAIL; @@ -844,8 +846,8 @@ int main(int argc, char *argv[]) if (ksm_write_sysfs(KSM_FP("run"), 2) || ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) || - numa_available() ? 0 : - ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) || + (numa_available() ? 0 : + ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1)) || ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count)) ksft_exit_fail_msg("Cannot set up KSM tunables\n"); diff --git a/tools/testing/selftests/mm/memory-failure.c b/tools/testing/selftests/mm/memory-failure.c index 032ed952057c62..1a5a32e22cce29 100644 --- a/tools/testing/selftests/mm/memory-failure.c +++ b/tools/testing/selftests/mm/memory-failure.c @@ -46,7 +46,7 @@ FIXTURE(memory_failure) unsigned long pfn; int pagemap_fd; int kpageflags_fd; - bool triggered; + bool injection_attempted; }; FIXTURE_VARIANT(memory_failure) @@ -122,13 +122,6 @@ static void teardown_sighandler(void) sigaction(SIGBUS, &sa, NULL); } -FIXTURE_TEARDOWN(memory_failure) -{ - close(self->kpageflags_fd); - close(self->pagemap_fd); - teardown_sighandler(); -} - static void prepare(struct __test_metadata *_metadata, FIXTURE_DATA(memory_failure) * self, void *vaddr) { @@ -200,8 +193,7 @@ static void check(struct __test_metadata *_metadata, FIXTURE_DATA(memory_failure ASSERT_EQ(pfn_flags & KPF_HWPOISON, KPF_HWPOISON); } -static void cleanup(struct __test_metadata *_metadata, FIXTURE_DATA(memory_failure) * self, - void *vaddr) +static void cleanup(struct __test_metadata *_metadata, FIXTURE_DATA(memory_failure) * self) { unsigned long size; uint64_t pfn_flags; @@ -217,6 +209,20 @@ static void cleanup(struct __test_metadata *_metadata, FIXTURE_DATA(memory_failu ASSERT_EQ(size, self->corrupted_size); } +FIXTURE_TEARDOWN(memory_failure) +{ + /* + * Injection may poison the page before failing or delivering SIGBUS, so + * clean up after every injection attempt. + */ + if (self->injection_attempted) + cleanup(_metadata, self); + + close(self->kpageflags_fd); + close(self->pagemap_fd); + teardown_sighandler(); +} + TEST_F(memory_failure, anon) { char *addr; @@ -231,8 +237,8 @@ TEST_F(memory_failure, anon) prepare(_metadata, self, addr); ret = sigsetjmp(signal_jmp_buf, 1); - if (!self->triggered) { - self->triggered = true; + if (!self->injection_attempted) { + self->injection_attempted = true; ASSERT_EQ(variant->inject(self, addr), 0); FORCE_READ(*addr); } @@ -242,8 +248,6 @@ TEST_F(memory_failure, anon) else check(_metadata, self, addr, MADV_SOFT_ANON, ret); - cleanup(_metadata, self, addr); - ASSERT_EQ(munmap(addr, self->page_size), 0); } @@ -296,8 +300,8 @@ TEST_F(memory_failure, clean_pagecache) prepare(_metadata, self, addr); ret = sigsetjmp(signal_jmp_buf, 1); - if (!self->triggered) { - self->triggered = true; + if (!self->injection_attempted) { + self->injection_attempted = true; ASSERT_EQ(variant->inject(self, addr), 0); FORCE_READ(*addr); } @@ -307,8 +311,6 @@ TEST_F(memory_failure, clean_pagecache) else check(_metadata, self, addr, MADV_SOFT_CLEAN_PAGECACHE, ret); - cleanup(_metadata, self, addr); - ASSERT_EQ(munmap(addr, self->page_size), 0); ASSERT_EQ(close(fd), 0); @@ -337,8 +339,8 @@ TEST_F(memory_failure, dirty_pagecache) prepare(_metadata, self, addr); ret = sigsetjmp(signal_jmp_buf, 1); - if (!self->triggered) { - self->triggered = true; + if (!self->injection_attempted) { + self->injection_attempted = true; ASSERT_EQ(variant->inject(self, addr), 0); FORCE_READ(*addr); } @@ -348,8 +350,6 @@ TEST_F(memory_failure, dirty_pagecache) else check(_metadata, self, addr, MADV_SOFT_DIRTY_PAGECACHE, ret); - cleanup(_metadata, self, addr); - ASSERT_EQ(munmap(addr, self->page_size), 0); ASSERT_EQ(close(fd), 0); diff --git a/tools/testing/selftests/mm/thuge-gen.c b/tools/testing/selftests/mm/thuge-gen.c index 22b9c2f1c35d23..50d0805b65db94 100644 --- a/tools/testing/selftests/mm/thuge-gen.c +++ b/tools/testing/selftests/mm/thuge-gen.c @@ -71,12 +71,16 @@ void test_mmap(unsigned long size, unsigned flags) void test_shmget(unsigned long size, unsigned flags) { - int id; - unsigned long before, after; + /* values for PAGE_SIZE test */ + unsigned long before = NUM_PAGES; + unsigned long after = 0; struct shm_info i; char *map; + int id; + + if (size != getpagesize()) + before = hugetlb_free_pages(size); - before = hugetlb_free_pages(size); id = shmget(IPC_PRIVATE, size * NUM_PAGES, IPC_CREAT|0600|flags); if (id < 0) { if (errno == EPERM) { @@ -97,10 +101,11 @@ void test_shmget(unsigned long size, unsigned flags) shmctl(id, IPC_RMID, NULL); memset(map, 0xff, size*NUM_PAGES); - after = hugetlb_free_pages(size); + if (size != getpagesize()) + after = hugetlb_free_pages(size); show(size); - ksft_test_result(size == getpagesize() || (before - after) == NUM_PAGES, + ksft_test_result((before - after) == NUM_PAGES, "%s: mmap %lu %x\n", __func__, size, flags); if (shmdt(map)) ksft_exit_fail_msg("%s: shmdt: %s\n", __func__, strerror(errno)); diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index 311fc5b4513eb4..2697d50d07f4fc 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -719,7 +719,7 @@ int read_file(const char *path, char *buf, size_t buflen) return (unsigned int) numread; } -void write_file(const char *path, const char *buf, size_t buflen) +static void __write_file(const char *path, const char *buf, size_t buflen, bool ignore_einval) { int fd, saved_errno; ssize_t numwritten; @@ -735,30 +735,48 @@ void write_file(const char *path, const char *buf, size_t buflen) saved_errno = errno; close(fd); errno = saved_errno; - if (numwritten < 0) + if (numwritten < 0) { + if (ignore_einval && errno == EINVAL) + return; ksft_exit_fail_msg("%s write(%.*s) failed: %s\n", path, (int)(buflen - 1), buf, strerror(errno)); + } if (numwritten != buflen - 1) ksft_exit_fail_msg("%s write(%.*s) is truncated, expected %zu bytes, got %zd bytes\n", path, (int)(buflen - 1), buf, buflen - 1, numwritten); } +void write_file(const char *path, const char *buf, size_t buflen) +{ + __write_file(path, buf, buflen, /* ignore_einval = */ false); +} + unsigned long read_num(const char *path) { char buf[21]; - if (read_file(path, buf, sizeof(buf)) < 0) + if (!read_file(path, buf, sizeof(buf))) ksft_exit_fail_perror("read_file()"); return strtoul(buf, NULL, 10); } -void write_num(const char *path, unsigned long num) +static void __write_num(const char *path, unsigned long num, bool ignore_einval) { char buf[21]; sprintf(buf, "%lu", num); - write_file(path, buf, strlen(buf) + 1); + __write_file(path, buf, strlen(buf) + 1, ignore_einval); +} + +void write_num(const char *path, unsigned long num) +{ + return __write_num(path, num, /* ignore_einval = */ false); +} + +void write_num_ignore_einval(const char *path, unsigned long num) +{ + return __write_num(path, num, /* ignore_einval = */ true); } static unsigned long shmall, shmmax; diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index ea8fc8fdf0eb0b..7799154b67eed0 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -168,6 +168,7 @@ void write_file(const char *path, const char *buf, size_t buflen); int read_file(const char *path, char *buf, size_t buflen); unsigned long read_num(const char *path); void write_num(const char *path, unsigned long num); +void write_num_ignore_einval(const char *path, unsigned long num); void shm_limits_prepare(unsigned long length); void __shm_limits_restore(void); diff --git a/tools/testing/selftests/namespaces/nsid_test.c b/tools/testing/selftests/namespaces/nsid_test.c index 46dc838cba82d2..a16f31f41d38fd 100644 --- a/tools/testing/selftests/namespaces/nsid_test.c +++ b/tools/testing/selftests/namespaces/nsid_test.c @@ -649,8 +649,6 @@ TEST_F(nsid, timens_separate) /* Fork a grandchild to actually enter the new namespace */ pid_t grandchild = fork(); if (grandchild == 0) { - /* Grandchild is in the new namespace */ - write(pipefd[1], "Y", 1); close(pipefd[1]); pause(); _exit(0); @@ -771,8 +769,6 @@ TEST_F(nsid, pidns_separate) /* Fork a grandchild to actually enter the new namespace */ pid_t grandchild = fork(); if (grandchild == 0) { - /* Grandchild is in the new namespace */ - write(pipefd[1], "Y", 1); close(pipefd[1]); pause(); _exit(0); diff --git a/tools/testing/selftests/proc/proc-maps-race.c b/tools/testing/selftests/proc/proc-maps-race.c index 1026d8c400e1ba..415eccb7046848 100644 --- a/tools/testing/selftests/proc/proc-maps-race.c +++ b/tools/testing/selftests/proc/proc-maps-race.c @@ -490,7 +490,8 @@ static bool query_addr_at(int maps_fd, void *addr, static inline bool split_vma(FIXTURE_DATA(proc_maps_race) *self) { - return mmap(self->mod_info->addr, self->page_size, self->mod_info->prot | PROT_EXEC, + /* PROT_NONE differs from both readable neighbors. */ + return mmap(self->mod_info->addr, self->page_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) != MAP_FAILED; } diff --git a/tools/testing/selftests/proc/proc-pidns.c b/tools/testing/selftests/proc/proc-pidns.c index 25b9a2933c4569..6f7c10fe97b304 100644 --- a/tools/testing/selftests/proc/proc-pidns.c +++ b/tools/testing/selftests/proc/proc-pidns.c @@ -6,6 +6,7 @@ #include #include +#include #include #include #include diff --git a/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h b/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h index d3133587d99682..5a29d6bec51f49 100644 --- a/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h +++ b/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h @@ -8,13 +8,11 @@ #ifndef _RSEQ_X86_THREAD_POINTER #define _RSEQ_X86_THREAD_POINTER -#include - #ifdef __cplusplus extern "C" { #endif -#if __GNUC_PREREQ (11, 1) +#if __GNUC__ > 11 || (__GNUC__ == 11 && __GNUC_MINOR__ >= 1) static inline void *rseq_thread_pointer(void) { return __builtin_thread_pointer(); diff --git a/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c b/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c index 6f4c3f5a1c5d99..7ef9de7b27ebfc 100644 --- a/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c +++ b/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c @@ -14,18 +14,16 @@ s32 BPF_STRUCT_OPS(ddsp_bogus_dsq_fail_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags) { s32 cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr, 0); + if (cpu < 0) + cpu = prev_cpu; - if (cpu >= 0) { - /* - * If we dispatch to a bogus DSQ that will fall back to the - * builtin global DSQ, we fail gracefully. - */ - scx_bpf_dsq_insert_vtime(p, 0xcafef00d, SCX_SLICE_DFL, - p->scx.dsq_vtime, 0); - return cpu; - } - - return prev_cpu; + /* + * If we dispatch to a bogus DSQ that will fall back to the + * builtin global DSQ, we fail gracefully. + */ + scx_bpf_dsq_insert_vtime(p, 0xcafef00d, SCX_SLICE_DFL, + p->scx.dsq_vtime, 0); + return cpu; } void BPF_STRUCT_OPS(ddsp_bogus_dsq_fail_exit, struct scx_exit_info *ei) diff --git a/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c b/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c index e4a55027778fd0..82dca4cdc0a6cb 100644 --- a/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c +++ b/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c @@ -14,15 +14,14 @@ s32 BPF_STRUCT_OPS(ddsp_vtimelocal_fail_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags) { s32 cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr, 0); + if (cpu < 0) + cpu = prev_cpu; - if (cpu >= 0) { - /* Shouldn't be allowed to vtime dispatch to a builtin DSQ. */ - scx_bpf_dsq_insert_vtime(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, - p->scx.dsq_vtime, 0); - return cpu; - } + /* Shouldn't be allowed to vtime dispatch to a builtin DSQ. */ + scx_bpf_dsq_insert_vtime(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, + p->scx.dsq_vtime, 0); - return prev_cpu; + return cpu; } void BPF_STRUCT_OPS(ddsp_vtimelocal_fail_exit, struct scx_exit_info *ei) diff --git a/tools/testing/selftests/sched_ext/exit.c b/tools/testing/selftests/sched_ext/exit.c index b987611789d16e..01b17092d5c8fe 100644 --- a/tools/testing/selftests/sched_ext/exit.c +++ b/tools/testing/selftests/sched_ext/exit.c @@ -31,6 +31,7 @@ static enum scx_test_status run(void *ctx) continue; skel = exit__open(); + SCX_FAIL_IF(!skel, "Failed to open"); SCX_ENUM_INIT(skel); skel->rodata->exit_point = tc; SCX_FAIL_IF(exit__load(skel), "Failed to load skel"); diff --git a/tools/testing/selftests/sched_ext/prog_run.c b/tools/testing/selftests/sched_ext/prog_run.c index 05974820ca69d8..1129ec2aaddc7f 100644 --- a/tools/testing/selftests/sched_ext/prog_run.c +++ b/tools/testing/selftests/sched_ext/prog_run.c @@ -28,7 +28,8 @@ static enum scx_test_status setup(void **ctx) static enum scx_test_status run(void *ctx) { struct prog_run *skel = ctx; - struct bpf_link *link; + struct bpf_link *link = NULL; + enum scx_test_status status = SCX_TEST_PASS; int prog_fd, err = 0; prog_fd = bpf_program__fd(skel->progs.prog_run_syscall); @@ -42,23 +43,40 @@ static enum scx_test_status run(void *ctx) link = bpf_map__attach_struct_ops(skel->maps.prog_run_ops); if (!link) { SCX_ERR("Failed to attach scheduler"); - close(prog_fd); - return SCX_TEST_FAIL; + status = SCX_TEST_FAIL; + goto out; } err = bpf_prog_test_run_opts(prog_fd, &topts); - SCX_EQ(err, 0); + if (err) { + SCX_ERR("BPF_PROG_RUN failed (%d)", err); + status = SCX_TEST_FAIL; + goto out; + } /* Assumes uei.kind is written last */ while (skel->data->uei.kind == EXIT_KIND(SCX_EXIT_NONE)) sched_yield(); - SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG_BPF)); - SCX_EQ(skel->data->uei.exit_code, 0xdeadbeef); + if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_UNREG_BPF)) { + SCX_ERR("Unexpected exit kind: %llu", + (unsigned long long)skel->data->uei.kind); + status = SCX_TEST_FAIL; + goto out; + } + if (skel->data->uei.exit_code != 0xdeadbeef) { + SCX_ERR("Unexpected exit code: %lld", + (long long)skel->data->uei.exit_code); + status = SCX_TEST_FAIL; + goto out; + } + +out: close(prog_fd); - bpf_link__destroy(link); + if (link) + bpf_link__destroy(link); - return SCX_TEST_PASS; + return status; } static void cleanup(void *ctx) diff --git a/tools/testing/selftests/timers/leap-a-day.c b/tools/testing/selftests/timers/leap-a-day.c index 3568cfb3e81577..97c8c66bea883c 100644 --- a/tools/testing/selftests/timers/leap-a-day.c +++ b/tools/testing/selftests/timers/leap-a-day.c @@ -9,16 +9,19 @@ * kernel's leap-second behavior, as well as how well applications * handle the leap-second discontinuity. * - * Usage: leap-a-day [-s] [-i ] + * Usage: leap-a-day [-w] [-i ] [-t] * * Options: - * -s: Each iteration, set the date to 10 seconds before midnight GMT. - * This speeds up the number of leapsecond transitions tested, - * but because it calls settimeofday frequently, advancing the - * time by 24 hours every ~16 seconds, it may cause application - * disruption. + * -w: Only set the leap-second flag and wait for the leap second + * each iteration, instead of advancing the time. By default the + * date is set to 10 seconds before midnight GMT, which speeds up + * the number of leapsecond transitions tested, but because it + * calls settimeofday frequently, advancing the time by 24 hours + * every ~16 seconds, it may cause application disruption. * - * -i: Number of iterations to run (default: infinite) + * -i: Number of iterations to run (-1 = infinite, default: 10) + * + * -t: Print TAI time. * * Other notes: Disabling NTP prior to running this is advised, as the two * may conflict in their commands to the kernel. @@ -186,7 +189,7 @@ int main(int argc, char **argv) int opt; /* Process arguments */ - while ((opt = getopt(argc, argv, "sti:")) != -1) { + while ((opt = getopt(argc, argv, "wti:")) != -1) { switch (opt) { case 'w': printf("Only setting leap-flag, not changing time. It could take up to a day for leap to trigger.\n"); diff --git a/tools/testing/selftests/vfio/lib/vfio_pci_device.c b/tools/testing/selftests/vfio/lib/vfio_pci_device.c index 94dc5fcecbeb63..3db65084a43510 100644 --- a/tools/testing/selftests/vfio/lib/vfio_pci_device.c +++ b/tools/testing/selftests/vfio/lib/vfio_pci_device.c @@ -30,13 +30,11 @@ static void vfio_pci_irq_set(struct vfio_pci_device *device, u32 index, u32 vector, u32 count, int *fds) { - u8 buf[sizeof(struct vfio_irq_set) + sizeof(int) * count]; - struct vfio_irq_set *irq = (void *)&buf; - int *irq_fds = (void *)&irq->data; + size_t argsz = sizeof(struct vfio_irq_set) + sizeof(int) * count; + struct vfio_irq_set *irq; - memset(buf, 0, sizeof(buf)); - - irq->argsz = sizeof(buf); + irq = calloc_assert(1, argsz); + irq->argsz = argsz; irq->flags = VFIO_IRQ_SET_ACTION_TRIGGER; irq->index = index; irq->start = vector; @@ -44,12 +42,13 @@ static void vfio_pci_irq_set(struct vfio_pci_device *device, if (count) { irq->flags |= VFIO_IRQ_SET_DATA_EVENTFD; - memcpy(irq_fds, fds, sizeof(int) * count); + memcpy(irq->data, fds, sizeof(int) * count); } else { irq->flags |= VFIO_IRQ_SET_DATA_NONE; } ioctl_assert(device->fd, VFIO_DEVICE_SET_IRQS, irq); + free(irq); } void vfio_pci_irq_trigger(struct vfio_pci_device *device, u32 index, u32 vector) @@ -118,15 +117,20 @@ static void vfio_pci_irq_get(struct vfio_pci_device *device, u32 index, static int vfio_device_feature_ioctl(int fd, u32 flags, void *data, size_t data_size) { - u8 buffer[sizeof(struct vfio_device_feature) + data_size] = {}; - struct vfio_device_feature *feature = (void *)buffer; + size_t argsz = sizeof(struct vfio_device_feature) + data_size; + struct vfio_device_feature *feature; + int ret; + feature = calloc_assert(1, argsz); memcpy(feature->data, data, data_size); - feature->argsz = sizeof(buffer); + feature->argsz = argsz; feature->flags = flags; - return ioctl(fd, VFIO_DEVICE_FEATURE, feature); + ret = ioctl(fd, VFIO_DEVICE_FEATURE, feature); + free(feature); + + return ret; } static void vfio_device_feature_set(int fd, u16 feature, void *data, size_t data_size) diff --git a/tools/testing/selftests/zram/zram_lib.sh b/tools/testing/selftests/zram/zram_lib.sh index 21ec1966de76ca..0d44d83888f9d4 100755 --- a/tools/testing/selftests/zram/zram_lib.sh +++ b/tools/testing/selftests/zram/zram_lib.sh @@ -37,7 +37,7 @@ kernel_gte() if [ $kernel_major -gt $major ]; then return 0 - elif [[ $kernel_major -eq $major && $kernel_minor -ge $minor ]]; then + elif [ $kernel_major -eq $major ] && [ $kernel_minor -ge $minor ]; then return 0 fi