diff --git a/patches/0001-cachyos-base-all.patch b/patches/0001-cachyos-base-all.patch index 7bd0007..3ed5a82 100644 --- a/patches/0001-cachyos-base-all.patch +++ b/patches/0001-cachyos-base-all.patch @@ -1,1048 +1,164 @@ -From bb47d9fb19e79f1962a974ae8dd30337c28713c4 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Sun, 2 Feb 2025 13:53:06 +0100 -Subject: [PATCH 01/12] amd-pstate +From e18c7382054960494788390ac3364676e797658f Mon Sep 17 00:00:00 2001 +From: Eric Naim +Date: Mon, 3 Feb 2025 11:04:37 +0800 +Subject: [PATCH 1/6] amd-pstate -Signed-off-by: Peter Jung +Signed-off-by: Eric Naim --- - drivers/cpufreq/amd-pstate-trace.h | 52 +++- - drivers/cpufreq/amd-pstate-ut.c | 12 +- - drivers/cpufreq/amd-pstate.c | 474 ++++++++++++++--------------- - drivers/cpufreq/amd-pstate.h | 3 - - 4 files changed, 276 insertions(+), 265 deletions(-) + kernel/sched/fair.c | 21 +++++++++++++++++++-- + kernel/sched/sched.h | 1 - + kernel/sched/topology.c | 15 +-------------- + 3 files changed, 20 insertions(+), 17 deletions(-) -diff --git a/drivers/cpufreq/amd-pstate-trace.h b/drivers/cpufreq/amd-pstate-trace.h -index 35f38ae67fb1..8d692415d905 100644 ---- a/drivers/cpufreq/amd-pstate-trace.h -+++ b/drivers/cpufreq/amd-pstate-trace.h -@@ -32,7 +32,6 @@ TRACE_EVENT(amd_pstate_perf, - u64 aperf, - u64 tsc, - unsigned int cpu_id, -- bool changed, - bool fast_switch - ), +diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c +index ce2e94ccad0c..f9464644186c 100644 +--- a/kernel/sched/fair.c ++++ b/kernel/sched/fair.c +@@ -9887,6 +9887,8 @@ struct sg_lb_stats { + unsigned int group_weight; + enum group_type group_type; + unsigned int group_asym_packing; /* Tasks should be moved to preferred CPU */ ++ unsigned int asym_prefer_cpu; /* Group CPU with highest asym priority */ ++ int highest_asym_prio; /* Asym priority of asym_prefer_cpu */ + unsigned int group_smt_balance; /* Task on busy SMT be moved */ + unsigned long group_misfit_task_load; /* A CPU has a task too big for its capacity */ + #ifdef CONFIG_NUMA_BALANCING +@@ -10216,7 +10218,7 @@ sched_group_asym(struct lb_env *env, struct sg_lb_stats *sgs, struct sched_group + (sgs->group_weight - sgs->idle_cpus != 1)) + return false; -@@ -44,7 +43,6 @@ TRACE_EVENT(amd_pstate_perf, - aperf, - tsc, - cpu_id, -- changed, - fast_switch - ), +- return sched_asym(env->sd, env->dst_cpu, group->asym_prefer_cpu); ++ return sched_asym(env->sd, env->dst_cpu, sgs->asym_prefer_cpu); + } -@@ -57,7 +55,6 @@ TRACE_EVENT(amd_pstate_perf, - __field(unsigned long long, aperf) - __field(unsigned long long, tsc) - __field(unsigned int, cpu_id) -- __field(bool, changed) - __field(bool, fast_switch) - ), + /* One group has more than one SMT CPU while the other group does not */ +@@ -10297,6 +10299,17 @@ sched_reduced_capacity(struct rq *rq, struct sched_domain *sd) + return check_cpu_capacity(rq, sd); + } -@@ -70,11 +67,10 @@ TRACE_EVENT(amd_pstate_perf, - __entry->aperf = aperf; - __entry->tsc = tsc; - __entry->cpu_id = cpu_id; -- __entry->changed = changed; - __entry->fast_switch = fast_switch; - ), - -- TP_printk("amd_min_perf=%lu amd_des_perf=%lu amd_max_perf=%lu freq=%llu mperf=%llu aperf=%llu tsc=%llu cpu_id=%u changed=%s fast_switch=%s", -+ TP_printk("amd_min_perf=%lu amd_des_perf=%lu amd_max_perf=%lu freq=%llu mperf=%llu aperf=%llu tsc=%llu cpu_id=%u fast_switch=%s", - (unsigned long)__entry->min_perf, - (unsigned long)__entry->target_perf, - (unsigned long)__entry->capacity, -@@ -83,11 +79,55 @@ TRACE_EVENT(amd_pstate_perf, - (unsigned long long)__entry->aperf, - (unsigned long long)__entry->tsc, - (unsigned int)__entry->cpu_id, -- (__entry->changed) ? "true" : "false", - (__entry->fast_switch) ? "true" : "false" - ) - ); - -+TRACE_EVENT(amd_pstate_epp_perf, ++static inline void ++update_sg_pick_asym_prefer(struct sg_lb_stats *sgs, int cpu) ++{ ++ int asym_prio = arch_asym_cpu_priority(cpu); + -+ TP_PROTO(unsigned int cpu_id, -+ unsigned int highest_perf, -+ unsigned int epp, -+ unsigned int min_perf, -+ unsigned int max_perf, -+ bool boost -+ ), ++ if (asym_prio > sgs->highest_asym_prio) { ++ sgs->asym_prefer_cpu = cpu; ++ sgs->highest_asym_prio = asym_prio; ++ } ++} + -+ TP_ARGS(cpu_id, -+ highest_perf, -+ epp, -+ min_perf, -+ max_perf, -+ boost), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, cpu_id) -+ __field(unsigned int, highest_perf) -+ __field(unsigned int, epp) -+ __field(unsigned int, min_perf) -+ __field(unsigned int, max_perf) -+ __field(bool, boost) -+ ), -+ -+ TP_fast_assign( -+ __entry->cpu_id = cpu_id; -+ __entry->highest_perf = highest_perf; -+ __entry->epp = epp; -+ __entry->min_perf = min_perf; -+ __entry->max_perf = max_perf; -+ __entry->boost = boost; -+ ), -+ -+ TP_printk("cpu%u: [%u<->%u]/%u, epp=%u, boost=%u", -+ (unsigned int)__entry->cpu_id, -+ (unsigned int)__entry->min_perf, -+ (unsigned int)__entry->max_perf, -+ (unsigned int)__entry->highest_perf, -+ (unsigned int)__entry->epp, -+ (bool)__entry->boost -+ ) -+); -+ - #endif /* _AMD_PSTATE_TRACE_H */ + /** + * update_sg_lb_stats - Update sched_group's statistics for load balancing. + * @env: The load balancing environment. +@@ -10319,6 +10332,7 @@ static inline void update_sg_lb_stats(struct lb_env *env, + memset(sgs, 0, sizeof(*sgs)); - /* This part must be outside protection */ -diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c -index a261d7300951..3a0a380c3590 100644 ---- a/drivers/cpufreq/amd-pstate-ut.c -+++ b/drivers/cpufreq/amd-pstate-ut.c -@@ -207,7 +207,6 @@ static void amd_pstate_ut_check_freq(u32 index) - int cpu = 0; - struct cpufreq_policy *policy = NULL; - struct amd_cpudata *cpudata = NULL; -- u32 nominal_freq_khz; + local_group = group == sds->local; ++ sgs->highest_asym_prio = INT_MIN; - for_each_possible_cpu(cpu) { - policy = cpufreq_cpu_get(cpu); -@@ -215,14 +214,13 @@ static void amd_pstate_ut_check_freq(u32 index) - break; - cpudata = policy->driver_data; + for_each_cpu_and(i, sched_group_span(group), env->cpus) { + struct rq *rq = cpu_rq(i); +@@ -10332,6 +10346,9 @@ static inline void update_sg_lb_stats(struct lb_env *env, + nr_running = rq->nr_running; + sgs->sum_nr_running += nr_running; -- nominal_freq_khz = cpudata->nominal_freq*1000; -- if (!((cpudata->max_freq >= nominal_freq_khz) && -- (nominal_freq_khz > cpudata->lowest_nonlinear_freq) && -+ if (!((cpudata->max_freq >= cpudata->nominal_freq) && -+ (cpudata->nominal_freq > cpudata->lowest_nonlinear_freq) && - (cpudata->lowest_nonlinear_freq > cpudata->min_freq) && - (cpudata->min_freq > 0))) { - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; - pr_err("%s cpu%d max=%d >= nominal=%d > lowest_nonlinear=%d > min=%d > 0, the formula is incorrect!\n", -- __func__, cpu, cpudata->max_freq, nominal_freq_khz, -+ __func__, cpu, cpudata->max_freq, cpudata->nominal_freq, - cpudata->lowest_nonlinear_freq, cpudata->min_freq); - goto skip_test; ++ if (sd_flags & SD_ASYM_PACKING) ++ update_sg_pick_asym_prefer(sgs, i); ++ + if (cpu_overutilized(i)) + *sg_overutilized = 1; + +@@ -10453,7 +10470,7 @@ static bool update_sd_pick_busiest(struct lb_env *env, + + case group_asym_packing: + /* Prefer to move from lowest priority CPU's work */ +- return sched_asym_prefer(sds->busiest->asym_prefer_cpu, sg->asym_prefer_cpu); ++ return sched_asym_prefer(busiest->asym_prefer_cpu, sgs->asym_prefer_cpu); + + case group_misfit_task: + /* +diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h +index 38e0e323dda2..f5cd42d1e90c 100644 +--- a/kernel/sched/sched.h ++++ b/kernel/sched/sched.h +@@ -2086,7 +2086,6 @@ struct sched_group { + unsigned int group_weight; + unsigned int cores; + struct sched_group_capacity *sgc; +- int asym_prefer_cpu; /* CPU of highest priority in group */ + int flags; + + /* +diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c +index c49aea8c1025..b61a261ee45b 100644 +--- a/kernel/sched/topology.c ++++ b/kernel/sched/topology.c +@@ -1302,7 +1302,7 @@ static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) + WARN_ON(!sg); + + do { +- int cpu, cores = 0, max_cpu = -1; ++ int cpu, cores = 0; + + sg->group_weight = cpumask_weight(sched_group_span(sg)); + +@@ -1314,19 +1314,6 @@ static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) + #endif } -@@ -236,13 +234,13 @@ static void amd_pstate_ut_check_freq(u32 index) - - if (cpudata->boost_supported) { - if ((policy->max == cpudata->max_freq) || -- (policy->max == nominal_freq_khz)) -+ (policy->max == cpudata->nominal_freq)) - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS; - else { - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; - pr_err("%s cpu%d policy_max=%d should be equal cpu_max=%d or cpu_nominal=%d !\n", - __func__, cpu, policy->max, cpudata->max_freq, -- nominal_freq_khz); -+ cpudata->nominal_freq); - goto skip_test; - } - } else { -diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c -index 66e5dfc711c0..2330903a8b45 100644 ---- a/drivers/cpufreq/amd-pstate.c -+++ b/drivers/cpufreq/amd-pstate.c -@@ -22,6 +22,7 @@ - - #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -+#include - #include - #include - #include -@@ -88,6 +89,11 @@ static bool cppc_enabled; - static bool amd_pstate_prefcore = true; - static struct quirk_entry *quirks; - -+#define AMD_CPPC_MAX_PERF_MASK GENMASK(7, 0) -+#define AMD_CPPC_MIN_PERF_MASK GENMASK(15, 8) -+#define AMD_CPPC_DES_PERF_MASK GENMASK(23, 16) -+#define AMD_CPPC_EPP_PERF_MASK GENMASK(31, 24) -+ - /* - * AMD Energy Preference Performance (EPP) - * The EPP is used in the CCLK DPM controller to drive -@@ -180,120 +186,145 @@ static inline int get_mode_idx_from_str(const char *str, size_t size) - static DEFINE_MUTEX(amd_pstate_limits_lock); - static DEFINE_MUTEX(amd_pstate_driver_lock); - --static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) -+static s16 msr_get_epp(struct amd_cpudata *cpudata) - { -- u64 epp; -+ u64 value; - int ret; - -- if (cpu_feature_enabled(X86_FEATURE_CPPC)) { -- if (!cppc_req_cached) { -- epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, -- &cppc_req_cached); -- if (epp) -- return epp; + sg->cores = cores; +- +- if (!(sd->flags & SD_ASYM_PACKING)) +- goto next; +- +- for_each_cpu(cpu, sched_group_span(sg)) { +- if (max_cpu < 0) +- max_cpu = cpu; +- else if (sched_asym_prefer(cpu, max_cpu)) +- max_cpu = cpu; - } -- epp = (cppc_req_cached >> 24) & 0xFF; -- } else { -- ret = cppc_get_epp_perf(cpudata->cpu, &epp); -- if (ret < 0) { -- pr_debug("Could not retrieve energy perf value (%d)\n", ret); -- return -EIO; -- } -+ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); -+ if (ret < 0) { -+ pr_debug("Could not retrieve energy perf value (%d)\n", ret); -+ return ret; - } - -- return (s16)(epp & 0xff); -+ return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, value); - } - --static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata) -+DEFINE_STATIC_CALL(amd_pstate_get_epp, msr_get_epp); -+ -+static inline s16 amd_pstate_get_epp(struct amd_cpudata *cpudata) - { -- s16 epp; -- int index = -EINVAL; -+ return static_call(amd_pstate_get_epp)(cpudata); -+} - -- epp = amd_pstate_get_epp(cpudata, 0); -- if (epp < 0) -- return epp; -+static s16 shmem_get_epp(struct amd_cpudata *cpudata) -+{ -+ u64 epp; -+ int ret; - -- switch (epp) { -- case AMD_CPPC_EPP_PERFORMANCE: -- index = EPP_INDEX_PERFORMANCE; -- break; -- case AMD_CPPC_EPP_BALANCE_PERFORMANCE: -- index = EPP_INDEX_BALANCE_PERFORMANCE; -- break; -- case AMD_CPPC_EPP_BALANCE_POWERSAVE: -- index = EPP_INDEX_BALANCE_POWERSAVE; -- break; -- case AMD_CPPC_EPP_POWERSAVE: -- index = EPP_INDEX_POWERSAVE; -- break; -- default: -- break; -+ ret = cppc_get_epp_perf(cpudata->cpu, &epp); -+ if (ret < 0) { -+ pr_debug("Could not retrieve energy perf value (%d)\n", ret); -+ return ret; - } - -- return index; -+ return (s16)(epp & 0xff); - } - --static void msr_update_perf(struct amd_cpudata *cpudata, u32 min_perf, -- u32 des_perf, u32 max_perf, bool fast_switch) -+static int msr_update_perf(struct amd_cpudata *cpudata, u32 min_perf, -+ u32 des_perf, u32 max_perf, u32 epp, bool fast_switch) - { -- if (fast_switch) -- wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached)); -- else -- wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, -- READ_ONCE(cpudata->cppc_req_cached)); -+ u64 value, prev; -+ -+ value = prev = READ_ONCE(cpudata->cppc_req_cached); -+ -+ value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK | -+ AMD_CPPC_DES_PERF_MASK | AMD_CPPC_EPP_PERF_MASK); -+ value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, max_perf); -+ value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, des_perf); -+ value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf); -+ value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); -+ -+ if (value == prev) -+ return 0; -+ -+ if (fast_switch) { -+ wrmsrl(MSR_AMD_CPPC_REQ, value); -+ return 0; -+ } else { -+ int ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); -+ -+ if (ret) -+ return ret; -+ } -+ -+ WRITE_ONCE(cpudata->cppc_req_cached, value); -+ WRITE_ONCE(cpudata->epp_cached, epp); -+ -+ return 0; - } - - DEFINE_STATIC_CALL(amd_pstate_update_perf, msr_update_perf); - --static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata, -+static inline int amd_pstate_update_perf(struct amd_cpudata *cpudata, - u32 min_perf, u32 des_perf, -- u32 max_perf, bool fast_switch) -+ u32 max_perf, u32 epp, -+ bool fast_switch) - { -- static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, -- max_perf, fast_switch); -+ return static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, -+ max_perf, epp, fast_switch); - } - --static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) -+static int msr_set_epp(struct amd_cpudata *cpudata, u32 epp) - { -+ u64 value, prev; - int ret; -- struct cppc_perf_ctrls perf_ctrls; - -- if (cpu_feature_enabled(X86_FEATURE_CPPC)) { -- u64 value = READ_ONCE(cpudata->cppc_req_cached); +- sg->asym_prefer_cpu = max_cpu; - -- value &= ~GENMASK_ULL(31, 24); -- value |= (u64)epp << 24; -- WRITE_ONCE(cpudata->cppc_req_cached, value); -+ value = prev = READ_ONCE(cpudata->cppc_req_cached); -+ value &= ~AMD_CPPC_EPP_PERF_MASK; -+ value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); +-next: + sg = sg->next; + } while (sg != sd->groups); -- ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); -- if (!ret) -- cpudata->epp_cached = epp; -- } else { -- amd_pstate_update_perf(cpudata, cpudata->min_limit_perf, 0U, -- cpudata->max_limit_perf, false); -+ if (value == prev) -+ return 0; - -- perf_ctrls.energy_perf = epp; -- ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); -- if (ret) { -- pr_debug("failed to set energy perf value (%d)\n", ret); -- return ret; -- } -- cpudata->epp_cached = epp; -+ ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); -+ if (ret) { -+ pr_err("failed to set energy perf value (%d)\n", ret); -+ return ret; - } - -+ /* update both so that msr_update_perf() can effectively check */ -+ WRITE_ONCE(cpudata->epp_cached, epp); -+ WRITE_ONCE(cpudata->cppc_req_cached, value); -+ - return ret; - } - --static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, -- int pref_index) -+DEFINE_STATIC_CALL(amd_pstate_set_epp, msr_set_epp); -+ -+static inline int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) -+{ -+ return static_call(amd_pstate_set_epp)(cpudata, epp); -+} -+ -+static int shmem_set_epp(struct amd_cpudata *cpudata, u32 epp) - { -- int epp = -EINVAL; - int ret; -+ struct cppc_perf_ctrls perf_ctrls; -+ -+ if (epp == cpudata->epp_cached) -+ return 0; -+ -+ perf_ctrls.energy_perf = epp; -+ ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); -+ if (ret) { -+ pr_debug("failed to set energy perf value (%d)\n", ret); -+ return ret; -+ } -+ WRITE_ONCE(cpudata->epp_cached, epp); -+ -+ return ret; -+} -+ -+static int amd_pstate_set_energy_pref_index(struct cpufreq_policy *policy, -+ int pref_index) -+{ -+ struct amd_cpudata *cpudata = policy->driver_data; -+ int epp; - - if (!pref_index) - epp = cpudata->epp_default; -- -- if (epp == -EINVAL) -+ else - epp = epp_values[pref_index]; - - if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { -@@ -301,9 +332,15 @@ static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, - return -EBUSY; - } - -- ret = amd_pstate_set_epp(cpudata, epp); -+ if (trace_amd_pstate_epp_perf_enabled()) { -+ trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, -+ epp, -+ FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cpudata->cppc_req_cached), -+ FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), -+ policy->boost_enabled); -+ } - -- return ret; -+ return amd_pstate_set_epp(cpudata, epp); - } - - static inline int msr_cppc_enable(bool enable) -@@ -442,17 +479,23 @@ static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) - return static_call(amd_pstate_init_perf)(cpudata); - } - --static void shmem_update_perf(struct amd_cpudata *cpudata, -- u32 min_perf, u32 des_perf, -- u32 max_perf, bool fast_switch) -+static int shmem_update_perf(struct amd_cpudata *cpudata, u32 min_perf, -+ u32 des_perf, u32 max_perf, u32 epp, bool fast_switch) - { - struct cppc_perf_ctrls perf_ctrls; - -+ if (cppc_state == AMD_PSTATE_ACTIVE) { -+ int ret = shmem_set_epp(cpudata, epp); -+ -+ if (ret) -+ return ret; -+ } -+ - perf_ctrls.max_perf = max_perf; - perf_ctrls.min_perf = min_perf; - perf_ctrls.desired_perf = des_perf; - -- cppc_set_perf(cpudata->cpu, &perf_ctrls); -+ return cppc_set_perf(cpudata->cpu, &perf_ctrls); - } - - static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) -@@ -493,14 +536,8 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, - { - unsigned long max_freq; - struct cpufreq_policy *policy = cpufreq_cpu_get(cpudata->cpu); -- u64 prev = READ_ONCE(cpudata->cppc_req_cached); - u32 nominal_perf = READ_ONCE(cpudata->nominal_perf); -- u64 value = prev; - -- min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf, -- cpudata->max_limit_perf); -- max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf, -- cpudata->max_limit_perf); - des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); - - max_freq = READ_ONCE(cpudata->max_limit_freq); -@@ -511,34 +548,18 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, - des_perf = 0; - } - -- value &= ~AMD_CPPC_MIN_PERF(~0L); -- value |= AMD_CPPC_MIN_PERF(min_perf); -- -- value &= ~AMD_CPPC_DES_PERF(~0L); -- value |= AMD_CPPC_DES_PERF(des_perf); -- - /* limit the max perf when core performance boost feature is disabled */ - if (!cpudata->boost_supported) - max_perf = min_t(unsigned long, nominal_perf, max_perf); - -- value &= ~AMD_CPPC_MAX_PERF(~0L); -- value |= AMD_CPPC_MAX_PERF(max_perf); -- - if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { - trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, - cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, -- cpudata->cpu, (value != prev), fast_switch); -+ cpudata->cpu, fast_switch); - } - -- if (value == prev) -- goto cpufreq_policy_put; -+ amd_pstate_update_perf(cpudata, min_perf, des_perf, max_perf, 0, fast_switch); - -- WRITE_ONCE(cpudata->cppc_req_cached, value); -- -- amd_pstate_update_perf(cpudata, min_perf, des_perf, -- max_perf, fast_switch); -- --cpufreq_policy_put: - cpufreq_cpu_put(policy); - } - -@@ -570,7 +591,7 @@ static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) - - static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) - { -- u32 max_limit_perf, min_limit_perf, lowest_perf, max_perf, max_freq; -+ u32 max_limit_perf, min_limit_perf, max_perf, max_freq; - struct amd_cpudata *cpudata = policy->driver_data; - - max_perf = READ_ONCE(cpudata->highest_perf); -@@ -578,12 +599,8 @@ static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) - max_limit_perf = div_u64(policy->max * max_perf, max_freq); - min_limit_perf = div_u64(policy->min * max_perf, max_freq); - -- lowest_perf = READ_ONCE(cpudata->lowest_perf); -- if (min_limit_perf < lowest_perf) -- min_limit_perf = lowest_perf; -- -- if (max_limit_perf < min_limit_perf) -- max_limit_perf = min_limit_perf; -+ if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) -+ min_limit_perf = min(cpudata->nominal_perf, max_limit_perf); - - WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf); - WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf); -@@ -704,8 +721,8 @@ static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) - - if (on) - policy->cpuinfo.max_freq = max_freq; -- else if (policy->cpuinfo.max_freq > nominal_freq * 1000) -- policy->cpuinfo.max_freq = nominal_freq * 1000; -+ else if (policy->cpuinfo.max_freq > nominal_freq) -+ policy->cpuinfo.max_freq = nominal_freq; - - policy->max = policy->cpuinfo.max_freq; - -@@ -727,12 +744,11 @@ static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) - pr_err("Boost mode is not supported by this processor or SBIOS\n"); - return -EOPNOTSUPP; - } -- mutex_lock(&amd_pstate_driver_lock); -+ guard(mutex)(&amd_pstate_driver_lock); -+ - ret = amd_pstate_cpu_boost_update(policy, state); -- WRITE_ONCE(cpudata->boost_state, !ret ? state : false); - policy->boost_enabled = !ret ? state : false; - refresh_frequency_limits(policy); -- mutex_unlock(&amd_pstate_driver_lock); - - return ret; - } -@@ -752,9 +768,6 @@ static int amd_pstate_init_boost_support(struct amd_cpudata *cpudata) - goto exit_err; - } - -- /* at least one CPU supports CPB, even if others fail later on to set up */ -- current_pstate_driver->boost_enabled = true; -- - ret = rdmsrl_on_cpu(cpudata->cpu, MSR_K7_HWCR, &boost_val); - if (ret) { - pr_err_once("failed to read initial CPU boost state!\n"); -@@ -802,7 +815,7 @@ static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) - * sched_set_itmt_support(true) has been called and it is valid to - * update them at any time after it has been called. - */ -- sched_set_itmt_core_prio((int)READ_ONCE(cpudata->highest_perf), cpudata->cpu); -+ sched_set_itmt_core_prio((int)READ_ONCE(cpudata->prefcore_ranking), cpudata->cpu); - - schedule_work(&sched_prefcore_work); - } -@@ -823,7 +836,8 @@ static void amd_pstate_update_limits(unsigned int cpu) - if (!amd_pstate_prefcore) - return; - -- mutex_lock(&amd_pstate_driver_lock); -+ guard(mutex)(&amd_pstate_driver_lock); -+ - ret = amd_get_highest_perf(cpu, &cur_high); - if (ret) - goto free_cpufreq_put; -@@ -843,7 +857,6 @@ static void amd_pstate_update_limits(unsigned int cpu) - if (!highest_perf_changed) - cpufreq_update_policy(cpu); - -- mutex_unlock(&amd_pstate_driver_lock); - } - - /* -@@ -905,29 +918,29 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) - return ret; - - if (quirks && quirks->lowest_freq) -- min_freq = quirks->lowest_freq * 1000; -+ min_freq = quirks->lowest_freq; - else -- min_freq = cppc_perf.lowest_freq * 1000; -+ min_freq = cppc_perf.lowest_freq; - - if (quirks && quirks->nominal_freq) -- nominal_freq = quirks->nominal_freq ; -+ nominal_freq = quirks->nominal_freq; - else - nominal_freq = cppc_perf.nominal_freq; - - nominal_perf = READ_ONCE(cpudata->nominal_perf); - - boost_ratio = div_u64(cpudata->highest_perf << SCHED_CAPACITY_SHIFT, nominal_perf); -- max_freq = (nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT) * 1000; -+ max_freq = (nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT); - - lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); - lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT, - nominal_perf); -- lowest_nonlinear_freq = (nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT) * 1000; -+ lowest_nonlinear_freq = (nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT); - -- WRITE_ONCE(cpudata->min_freq, min_freq); -- WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq); -- WRITE_ONCE(cpudata->nominal_freq, nominal_freq); -- WRITE_ONCE(cpudata->max_freq, max_freq); -+ WRITE_ONCE(cpudata->min_freq, min_freq * 1000); -+ WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq * 1000); -+ WRITE_ONCE(cpudata->nominal_freq, nominal_freq * 1000); -+ WRITE_ONCE(cpudata->max_freq, max_freq * 1000); - - /** - * Below values need to be initialized correctly, otherwise driver will fail to load -@@ -937,13 +950,13 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) - */ - if (min_freq <= 0 || max_freq <= 0 || nominal_freq <= 0 || min_freq > max_freq) { - pr_err("min_freq(%d) or max_freq(%d) or nominal_freq(%d) value is incorrect\n", -- min_freq, max_freq, nominal_freq * 1000); -+ min_freq, max_freq, nominal_freq); - return -EINVAL; - } - -- if (lowest_nonlinear_freq <= min_freq || lowest_nonlinear_freq > nominal_freq * 1000) { -+ if (lowest_nonlinear_freq <= min_freq || lowest_nonlinear_freq > nominal_freq) { - pr_err("lowest_nonlinear_freq(%d) value is out of range [min_freq(%d), nominal_freq(%d)]\n", -- lowest_nonlinear_freq, min_freq, nominal_freq * 1000); -+ lowest_nonlinear_freq, min_freq, nominal_freq); - return -EINVAL; - } - -@@ -1160,7 +1173,6 @@ static ssize_t show_energy_performance_available_preferences( - static ssize_t store_energy_performance_preference( - struct cpufreq_policy *policy, const char *buf, size_t count) - { -- struct amd_cpudata *cpudata = policy->driver_data; - char str_preference[21]; - ssize_t ret; - -@@ -1172,11 +1184,11 @@ static ssize_t store_energy_performance_preference( - if (ret < 0) - return -EINVAL; - -- mutex_lock(&amd_pstate_limits_lock); -- ret = amd_pstate_set_energy_pref_index(cpudata, ret); -- mutex_unlock(&amd_pstate_limits_lock); -+ guard(mutex)(&amd_pstate_limits_lock); -+ -+ ret = amd_pstate_set_energy_pref_index(policy, ret); - -- return ret ?: count; -+ return ret ? ret : count; - } - - static ssize_t show_energy_performance_preference( -@@ -1185,9 +1197,22 @@ static ssize_t show_energy_performance_preference( - struct amd_cpudata *cpudata = policy->driver_data; - int preference; - -- preference = amd_pstate_get_energy_pref_index(cpudata); -- if (preference < 0) -- return preference; -+ switch (cpudata->epp_cached) { -+ case AMD_CPPC_EPP_PERFORMANCE: -+ preference = EPP_INDEX_PERFORMANCE; -+ break; -+ case AMD_CPPC_EPP_BALANCE_PERFORMANCE: -+ preference = EPP_INDEX_BALANCE_PERFORMANCE; -+ break; -+ case AMD_CPPC_EPP_BALANCE_POWERSAVE: -+ preference = EPP_INDEX_BALANCE_POWERSAVE; -+ break; -+ case AMD_CPPC_EPP_POWERSAVE: -+ preference = EPP_INDEX_POWERSAVE; -+ break; -+ default: -+ return -EINVAL; -+ } - - return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); - } -@@ -1236,6 +1261,9 @@ static int amd_pstate_register_driver(int mode) - return ret; - } - -+ /* at least one CPU supports CPB */ -+ current_pstate_driver->boost_enabled = cpu_feature_enabled(X86_FEATURE_CPB); -+ - ret = cpufreq_register_driver(current_pstate_driver); - if (ret) { - amd_pstate_driver_cleanup(); -@@ -1340,13 +1368,10 @@ EXPORT_SYMBOL_GPL(amd_pstate_update_status); - static ssize_t status_show(struct device *dev, - struct device_attribute *attr, char *buf) - { -- ssize_t ret; - -- mutex_lock(&amd_pstate_driver_lock); -- ret = amd_pstate_show_status(buf); -- mutex_unlock(&amd_pstate_driver_lock); -+ guard(mutex)(&amd_pstate_driver_lock); - -- return ret; -+ return amd_pstate_show_status(buf); - } - - static ssize_t status_store(struct device *a, struct device_attribute *b, -@@ -1355,9 +1380,8 @@ static ssize_t status_store(struct device *a, struct device_attribute *b, - char *p = memchr(buf, '\n', count); - int ret; - -- mutex_lock(&amd_pstate_driver_lock); -+ guard(mutex)(&amd_pstate_driver_lock); - ret = amd_pstate_update_status(buf, p ? p - buf : count); -- mutex_unlock(&amd_pstate_driver_lock); - - return ret < 0 ? ret : count; - } -@@ -1451,7 +1475,6 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) - return -ENOMEM; - - cpudata->cpu = policy->cpu; -- cpudata->epp_policy = 0; - - ret = amd_pstate_init_perf(cpudata); - if (ret) -@@ -1477,8 +1500,6 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) - - policy->driver_data = cpudata; - -- cpudata->epp_cached = cpudata->epp_default = amd_pstate_get_epp(cpudata, 0); -- - policy->min = policy->cpuinfo.min_freq; - policy->max = policy->cpuinfo.max_freq; - -@@ -1489,10 +1510,13 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) - * the default cpufreq governor is neither powersave nor performance. - */ - if (amd_pstate_acpi_pm_profile_server() || -- amd_pstate_acpi_pm_profile_undefined()) -+ amd_pstate_acpi_pm_profile_undefined()) { - policy->policy = CPUFREQ_POLICY_PERFORMANCE; -- else -+ cpudata->epp_default = amd_pstate_get_epp(cpudata); -+ } else { - policy->policy = CPUFREQ_POLICY_POWERSAVE; -+ cpudata->epp_default = AMD_CPPC_EPP_BALANCE_PERFORMANCE; -+ } - - if (cpu_feature_enabled(X86_FEATURE_CPPC)) { - ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); -@@ -1505,6 +1529,9 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) - return ret; - WRITE_ONCE(cpudata->cppc_cap1_cached, value); - } -+ ret = amd_pstate_set_epp(cpudata, cpudata->epp_default); -+ if (ret) -+ return ret; - - current_pstate_driver->adjust_perf = NULL; - -@@ -1530,51 +1557,24 @@ static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) - static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) - { - struct amd_cpudata *cpudata = policy->driver_data; -- u32 max_perf, min_perf; -- u64 value; -- s16 epp; -+ u32 epp; - -- max_perf = READ_ONCE(cpudata->highest_perf); -- min_perf = READ_ONCE(cpudata->lowest_perf); - amd_pstate_update_min_max_limit(policy); - -- max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf, -- cpudata->max_limit_perf); -- min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf, -- cpudata->max_limit_perf); -- value = READ_ONCE(cpudata->cppc_req_cached); -- - if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) -- min_perf = min(cpudata->nominal_perf, max_perf); -- -- /* Initial min/max values for CPPC Performance Controls Register */ -- value &= ~AMD_CPPC_MIN_PERF(~0L); -- value |= AMD_CPPC_MIN_PERF(min_perf); -- -- value &= ~AMD_CPPC_MAX_PERF(~0L); -- value |= AMD_CPPC_MAX_PERF(max_perf); -- -- /* CPPC EPP feature require to set zero to the desire perf bit */ -- value &= ~AMD_CPPC_DES_PERF(~0L); -- value |= AMD_CPPC_DES_PERF(0); -- -- cpudata->epp_policy = cpudata->policy; -+ epp = 0; -+ else -+ epp = READ_ONCE(cpudata->epp_cached); - -- /* Get BIOS pre-defined epp value */ -- epp = amd_pstate_get_epp(cpudata, value); -- if (epp < 0) { -- /** -- * This return value can only be negative for shared_memory -- * systems where EPP register read/write not supported. -- */ -- return epp; -+ if (trace_amd_pstate_epp_perf_enabled()) { -+ trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, epp, -+ cpudata->min_limit_perf, -+ cpudata->max_limit_perf, -+ policy->boost_enabled); - } - -- if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) -- epp = 0; -- -- WRITE_ONCE(cpudata->cppc_req_cached, value); -- return amd_pstate_set_epp(cpudata, epp); -+ return amd_pstate_update_perf(cpudata, cpudata->min_limit_perf, 0U, -+ cpudata->max_limit_perf, epp, false); - } - - static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) -@@ -1603,87 +1603,63 @@ static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) - return 0; - } - --static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) -+static int amd_pstate_epp_reenable(struct cpufreq_policy *policy) - { -- struct cppc_perf_ctrls perf_ctrls; -- u64 value, max_perf; -+ struct amd_cpudata *cpudata = policy->driver_data; -+ u64 max_perf; - int ret; - - ret = amd_pstate_cppc_enable(true); - if (ret) - pr_err("failed to enable amd pstate during resume, return %d\n", ret); - -- value = READ_ONCE(cpudata->cppc_req_cached); - max_perf = READ_ONCE(cpudata->highest_perf); - -- if (cpu_feature_enabled(X86_FEATURE_CPPC)) { -- wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); -- } else { -- perf_ctrls.max_perf = max_perf; -- cppc_set_perf(cpudata->cpu, &perf_ctrls); -- perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); -- cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); -+ if (trace_amd_pstate_epp_perf_enabled()) { -+ trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, -+ cpudata->epp_cached, -+ FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cpudata->cppc_req_cached), -+ max_perf, policy->boost_enabled); - } -+ -+ return amd_pstate_update_perf(cpudata, 0, 0, max_perf, cpudata->epp_cached, false); - } - - static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) - { - struct amd_cpudata *cpudata = policy->driver_data; -+ int ret; - - pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); - -- if (cppc_state == AMD_PSTATE_ACTIVE) { -- amd_pstate_epp_reenable(cpudata); -- cpudata->suspended = false; -- } -+ ret = amd_pstate_epp_reenable(policy); -+ if (ret) -+ return ret; -+ cpudata->suspended = false; - - return 0; - } - --static void amd_pstate_epp_offline(struct cpufreq_policy *policy) --{ -- struct amd_cpudata *cpudata = policy->driver_data; -- struct cppc_perf_ctrls perf_ctrls; -- int min_perf; -- u64 value; -- -- min_perf = READ_ONCE(cpudata->lowest_perf); -- value = READ_ONCE(cpudata->cppc_req_cached); -- -- mutex_lock(&amd_pstate_limits_lock); -- if (cpu_feature_enabled(X86_FEATURE_CPPC)) { -- cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; -- -- /* Set max perf same as min perf */ -- value &= ~AMD_CPPC_MAX_PERF(~0L); -- value |= AMD_CPPC_MAX_PERF(min_perf); -- value &= ~AMD_CPPC_MIN_PERF(~0L); -- value |= AMD_CPPC_MIN_PERF(min_perf); -- wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); -- } else { -- perf_ctrls.desired_perf = 0; -- perf_ctrls.min_perf = min_perf; -- perf_ctrls.max_perf = min_perf; -- cppc_set_perf(cpudata->cpu, &perf_ctrls); -- perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE); -- cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); -- } -- mutex_unlock(&amd_pstate_limits_lock); --} -- - static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) - { - struct amd_cpudata *cpudata = policy->driver_data; -- -- pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu); -+ int min_perf; - - if (cpudata->suspended) - return 0; - -- if (cppc_state == AMD_PSTATE_ACTIVE) -- amd_pstate_epp_offline(policy); -+ min_perf = READ_ONCE(cpudata->lowest_perf); - -- return 0; -+ guard(mutex)(&amd_pstate_limits_lock); -+ -+ if (trace_amd_pstate_epp_perf_enabled()) { -+ trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, -+ AMD_CPPC_EPP_BALANCE_POWERSAVE, -+ min_perf, min_perf, policy->boost_enabled); -+ } -+ -+ return amd_pstate_update_perf(cpudata, min_perf, 0, min_perf, -+ AMD_CPPC_EPP_BALANCE_POWERSAVE, false); - } - - static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) -@@ -1711,12 +1687,10 @@ static int amd_pstate_epp_resume(struct cpufreq_policy *policy) - struct amd_cpudata *cpudata = policy->driver_data; - - if (cpudata->suspended) { -- mutex_lock(&amd_pstate_limits_lock); -+ guard(mutex)(&amd_pstate_limits_lock); - - /* enable amd pstate from suspend state*/ -- amd_pstate_epp_reenable(cpudata); -- -- mutex_unlock(&amd_pstate_limits_lock); -+ amd_pstate_epp_reenable(policy); - - cpudata->suspended = false; - } -@@ -1869,6 +1843,8 @@ static int __init amd_pstate_init(void) - static_call_update(amd_pstate_cppc_enable, shmem_cppc_enable); - static_call_update(amd_pstate_init_perf, shmem_init_perf); - static_call_update(amd_pstate_update_perf, shmem_update_perf); -+ static_call_update(amd_pstate_get_epp, shmem_get_epp); -+ static_call_update(amd_pstate_set_epp, shmem_set_epp); - } - - if (amd_pstate_prefcore) { -diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h -index cd573bc6b6db..9747e3be6cee 100644 ---- a/drivers/cpufreq/amd-pstate.h -+++ b/drivers/cpufreq/amd-pstate.h -@@ -57,7 +57,6 @@ struct amd_aperf_mperf { - * @hw_prefcore: check whether HW supports preferred core featue. - * Only when hw_prefcore and early prefcore param are true, - * AMD P-State driver supports preferred core featue. -- * @epp_policy: Last saved policy used to set energy-performance preference - * @epp_cached: Cached CPPC energy-performance preference value - * @policy: Cpufreq policy value - * @cppc_cap1_cached Cached MSR_AMD_CPPC_CAP1 register value -@@ -94,13 +93,11 @@ struct amd_cpudata { - bool hw_prefcore; - - /* EPP feature related attributes*/ -- s16 epp_policy; - s16 epp_cached; - u32 policy; - u64 cppc_cap1_cached; - bool suspended; - s16 epp_default; -- bool boost_state; - }; - - /* -- -2.48.0.rc1 +2.48.1 -From 24387ecea1f69a77a6236c4584dc06ea79f4ef1b Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Sun, 2 Feb 2025 13:53:18 +0100 -Subject: [PATCH 02/12] amd-tlb-broadcast +From 2796f641fb0fceaca8fabced62e297859b4cbfc4 Mon Sep 17 00:00:00 2001 +From: Eric Naim +Date: Mon, 3 Feb 2025 12:01:08 +0800 +Subject: [PATCH 2/6] amd-tlb-broadcast -Signed-off-by: Peter Jung +Signed-off-by: Eric Naim --- arch/x86/Kconfig | 2 +- arch/x86/Kconfig.cpu | 5 + arch/x86/hyperv/mmu.c | 1 - arch/x86/include/asm/cpufeatures.h | 1 + - arch/x86/include/asm/invlpgb.h | 107 +++++ - arch/x86/include/asm/mmu.h | 8 + - arch/x86/include/asm/mmu_context.h | 15 + + arch/x86/include/asm/invlpgb.h | 107 ++++++ + arch/x86/include/asm/mmu.h | 6 + + arch/x86/include/asm/mmu_context.h | 14 + arch/x86/include/asm/msr-index.h | 2 + arch/x86/include/asm/paravirt.h | 5 - arch/x86/include/asm/paravirt_types.h | 2 - arch/x86/include/asm/tlbbatch.h | 1 + - arch/x86/include/asm/tlbflush.h | 94 ++++- - arch/x86/kernel/alternative.c | 10 +- + arch/x86/include/asm/tlbflush.h | 93 ++++- arch/x86/kernel/cpu/amd.c | 12 + arch/x86/kernel/kvm.c | 1 - - arch/x86/kernel/paravirt.c | 6 - - arch/x86/mm/pgtable.c | 16 +- - arch/x86/mm/tlb.c | 540 ++++++++++++++++++++++--- + arch/x86/kernel/paravirt.c | 16 - + arch/x86/mm/pgtable.c | 27 +- + arch/x86/mm/tlb.c | 481 +++++++++++++++++++++++-- arch/x86/xen/mmu_pv.c | 1 - - include/linux/mm_types.h | 1 + - mm/memory.c | 1 - - mm/mmap.c | 2 - - mm/swap_state.c | 1 - - mm/vma.c | 2 - tools/arch/x86/include/asm/msr-index.h | 2 + - 25 files changed, 741 insertions(+), 97 deletions(-) + 19 files changed, 685 insertions(+), 94 deletions(-) create mode 100644 arch/x86/include/asm/invlpgb.h diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig -index ef6cfea9df73..1f824dcab4dc 100644 +index 87198d957e2f..17197d395976 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig -@@ -273,7 +273,7 @@ config X86 +@@ -277,7 +277,7 @@ config X86 select HAVE_PCI select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP @@ -1075,17 +191,17 @@ index 2a7279d80460..abe013a1b076 100644 This enables detection, tunings and quirks for AMD processors diff --git a/arch/x86/hyperv/mmu.c b/arch/x86/hyperv/mmu.c -index 1cc113200ff5..cbe6c71e17c1 100644 +index cc8c3bd0e7c2..1f7c3082a36d 100644 --- a/arch/x86/hyperv/mmu.c +++ b/arch/x86/hyperv/mmu.c -@@ -240,5 +240,4 @@ void hyperv_setup_mmu_ops(void) +@@ -239,5 +239,4 @@ void hyperv_setup_mmu_ops(void) pr_info("Using hypercall for remote TLB flush\n"); pv_ops.mmu.flush_tlb_multi = hyperv_flush_tlb_multi; - pv_ops.mmu.tlb_remove_table = tlb_remove_table; } diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h -index 645aa360628d..989e4c9cad2e 100644 +index 508c0dad116b..b5c66b7465ba 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -338,6 +338,7 @@ @@ -1210,19 +326,10 @@ index 000000000000..5fba41671a6d + +#endif /* _ASM_X86_INVLPGB */ diff --git a/arch/x86/include/asm/mmu.h b/arch/x86/include/asm/mmu.h -index ce4677b8b735..d71cd599fec4 100644 +index 3b496cdcb74b..d71cd599fec4 100644 --- a/arch/x86/include/asm/mmu.h +++ b/arch/x86/include/asm/mmu.h -@@ -37,6 +37,8 @@ typedef struct { - */ - atomic64_t tlb_gen; - -+ unsigned long next_trim_cpumask; -+ - #ifdef CONFIG_MODIFY_LDT_SYSCALL - struct rw_semaphore ldt_usr_sem; - struct ldt_struct *ldt; -@@ -67,6 +69,12 @@ typedef struct { +@@ -69,6 +69,12 @@ typedef struct { u16 pkey_allocation_map; s16 execute_only_pkey; #endif @@ -1236,7 +343,7 @@ index ce4677b8b735..d71cd599fec4 100644 #define INIT_MM_CONTEXT(mm) \ diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h -index 2886cb668d7f..d670699d32c2 100644 +index 795fdd53bd0a..d670699d32c2 100644 --- a/arch/x86/include/asm/mmu_context.h +++ b/arch/x86/include/asm/mmu_context.h @@ -139,6 +139,8 @@ static inline void mm_reset_untag_mask(struct mm_struct *mm) @@ -1248,15 +355,7 @@ index 2886cb668d7f..d670699d32c2 100644 /* * Init a new mm. Used on mm copies, like at fork() * and on mm's that are brand-new, like at execve(). -@@ -151,6 +153,7 @@ static inline int init_new_context(struct task_struct *tsk, - - mm->context.ctx_id = atomic64_inc_return(&last_mm_ctx_id); - atomic64_set(&mm->context.tlb_gen, 0); -+ mm->context.next_trim_cpumask = jiffies + HZ; - - #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS - if (cpu_feature_enabled(X86_FEATURE_OSPKE)) { -@@ -160,6 +163,14 @@ static inline int init_new_context(struct task_struct *tsk, +@@ -161,6 +163,14 @@ static inline int init_new_context(struct task_struct *tsk, mm->context.execute_only_pkey = -1; } #endif @@ -1271,7 +370,7 @@ index 2886cb668d7f..d670699d32c2 100644 mm_reset_untag_mask(mm); init_new_context_ldt(mm); return 0; -@@ -169,6 +180,10 @@ static inline int init_new_context(struct task_struct *tsk, +@@ -170,6 +180,10 @@ static inline int init_new_context(struct task_struct *tsk, static inline void destroy_context(struct mm_struct *mm) { destroy_context_ldt(mm); @@ -1283,7 +382,7 @@ index 2886cb668d7f..d670699d32c2 100644 extern void switch_mm(struct mm_struct *prev, struct mm_struct *next, diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h -index 3ae84c3b8e6d..dc1c1057f26e 100644 +index 9a71880eec07..a7ea9720ba3c 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -25,6 +25,7 @@ @@ -1303,7 +402,7 @@ index 3ae84c3b8e6d..dc1c1057f26e 100644 /* diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h -index d4eb9e1d61b8..794ba3647c6c 100644 +index 041aff51eb50..38a632a282d4 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -91,11 +91,6 @@ static inline void __flush_tlb_multi(const struct cpumask *cpumask, @@ -1319,10 +418,10 @@ index d4eb9e1d61b8..794ba3647c6c 100644 { PVOP_VCALL1(mmu.exit_mmap, mm); diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h -index 8d4fbe1be489..13405959e4db 100644 +index fea56b04f436..e26633c00455 100644 --- a/arch/x86/include/asm/paravirt_types.h +++ b/arch/x86/include/asm/paravirt_types.h -@@ -136,8 +136,6 @@ struct pv_mmu_ops { +@@ -134,8 +134,6 @@ struct pv_mmu_ops { void (*flush_tlb_multi)(const struct cpumask *cpus, const struct flush_tlb_info *info); @@ -1344,7 +443,7 @@ index 1ad56eb3e8a8..f9a17edf63ad 100644 #endif /* _ARCH_X86_TLBBATCH_H */ diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h -index 69e79fff41b8..f8aaa4bcb4d8 100644 +index 02fc2aa06e9e..f8aaa4bcb4d8 100644 --- a/arch/x86/include/asm/tlbflush.h +++ b/arch/x86/include/asm/tlbflush.h @@ -6,10 +6,12 @@ @@ -1374,15 +473,7 @@ index 69e79fff41b8..f8aaa4bcb4d8 100644 extern void initialize_tlbstate_and_flush(void); /* -@@ -222,6 +231,7 @@ struct flush_tlb_info { - unsigned int initiating_cpu; - u8 stride_shift; - u8 freed_tables; -+ u8 trim_cpumask; - }; - - void flush_tlb_local(void); -@@ -230,6 +240,78 @@ void flush_tlb_one_kernel(unsigned long addr); +@@ -231,6 +240,78 @@ void flush_tlb_one_kernel(unsigned long addr); void flush_tlb_multi(const struct cpumask *cpumask, const struct flush_tlb_info *info); @@ -1461,7 +552,7 @@ index 69e79fff41b8..f8aaa4bcb4d8 100644 #ifdef CONFIG_PARAVIRT #include #endif -@@ -277,21 +359,15 @@ static inline u64 inc_mm_tlb_gen(struct mm_struct *mm) +@@ -278,21 +359,15 @@ static inline u64 inc_mm_tlb_gen(struct mm_struct *mm) return atomic64_inc_return(&mm->context.tlb_gen); } @@ -1486,41 +577,8 @@ index 69e79fff41b8..f8aaa4bcb4d8 100644 static inline bool pte_flags_need_flush(unsigned long oldflags, unsigned long newflags, -diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c -index 243843e44e89..c71b575bf229 100644 ---- a/arch/x86/kernel/alternative.c -+++ b/arch/x86/kernel/alternative.c -@@ -1854,11 +1854,18 @@ static inline temp_mm_state_t use_temporary_mm(struct mm_struct *mm) - return temp_state; - } - -+__ro_after_init struct mm_struct *poking_mm; -+__ro_after_init unsigned long poking_addr; -+ - static inline void unuse_temporary_mm(temp_mm_state_t prev_state) - { - lockdep_assert_irqs_disabled(); -+ - switch_mm_irqs_off(NULL, prev_state.mm, current); - -+ /* Clear the cpumask, to indicate no TLB flushing is needed anywhere */ -+ cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(poking_mm)); -+ - /* - * Restore the breakpoints if they were disabled before the temporary mm - * was loaded. -@@ -1867,9 +1874,6 @@ static inline void unuse_temporary_mm(temp_mm_state_t prev_state) - hw_breakpoint_restore(); - } - --__ro_after_init struct mm_struct *poking_mm; --__ro_after_init unsigned long poking_addr; -- - static void text_poke_memcpy(void *dst, const void *src, size_t len) - { - memcpy(dst, src, len); diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c -index 79d2e17f6582..21076252a491 100644 +index 54194f5995de..38f454671c88 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -29,6 +29,8 @@ @@ -1532,7 +590,7 @@ index 79d2e17f6582..21076252a491 100644 static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p) { u32 gprs[8] = { 0 }; -@@ -1069,6 +1071,10 @@ static void init_amd(struct cpuinfo_x86 *c) +@@ -1073,6 +1075,10 @@ static void init_amd(struct cpuinfo_x86 *c) /* AMD CPUs don't need fencing after x2APIC/TSC_DEADLINE MSR writes. */ clear_cpu_cap(c, X86_FEATURE_APIC_MSRS_FENCE); @@ -1543,7 +601,7 @@ index 79d2e17f6582..21076252a491 100644 } #ifdef CONFIG_X86_32 -@@ -1135,6 +1141,12 @@ static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c) +@@ -1139,6 +1145,12 @@ static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c) tlb_lli_2m[ENTRIES] = eax & mask; tlb_lli_4m[ENTRIES] = tlb_lli_2m[ENTRIES] >> 1; @@ -1557,7 +615,7 @@ index 79d2e17f6582..21076252a491 100644 static const struct cpu_dev amd_cpu_dev = { diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c -index 21e9e4845354..83b7679658b1 100644 +index 7a422a6c5983..3be9b3342c67 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -838,7 +838,6 @@ static void __init kvm_guest_init(void) @@ -1569,22 +627,32 @@ index 21e9e4845354..83b7679658b1 100644 } diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c -index fec381533555..c019771e0123 100644 +index 1ccaa3397a67..2aa251d0b308 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c -@@ -59,11 +59,6 @@ void __init native_pv_lock_init(void) +@@ -59,21 +59,6 @@ void __init native_pv_lock_init(void) static_branch_enable(&virt_spin_lock_key); } +-#ifndef CONFIG_PT_RECLAIM -static void native_tlb_remove_table(struct mmu_gather *tlb, void *table) -{ -- tlb_remove_page(tlb, table); +- struct ptdesc *ptdesc = (struct ptdesc *)table; +- +- pagetable_dtor(ptdesc); +- tlb_remove_page(tlb, ptdesc_page(ptdesc)); -} +-#else +-static void native_tlb_remove_table(struct mmu_gather *tlb, void *table) +-{ +- tlb_remove_table(tlb, table); +-} +-#endif - struct static_key paravirt_steal_enabled; struct static_key paravirt_steal_rq_enabled; -@@ -191,7 +186,6 @@ struct paravirt_patch_template pv_ops = { +@@ -195,7 +180,6 @@ struct paravirt_patch_template pv_ops = { .mmu.flush_tlb_kernel = native_flush_tlb_global, .mmu.flush_tlb_one_user = native_flush_tlb_one_user, .mmu.flush_tlb_multi = native_flush_tlb_multi, @@ -1593,47 +661,57 @@ index fec381533555..c019771e0123 100644 .mmu.exit_mmap = paravirt_nop, .mmu.notify_page_enc_status_changed = paravirt_nop, diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c -index 5745a354a241..3dc4af1f7868 100644 +index 1fef5ad32d5a..8d1d228b096c 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c -@@ -18,14 +18,6 @@ EXPORT_SYMBOL(physical_mask); +@@ -18,25 +18,6 @@ EXPORT_SYMBOL(physical_mask); #define PGTABLE_HIGHMEM 0 #endif -#ifndef CONFIG_PARAVIRT +-#ifndef CONFIG_PT_RECLAIM -static inline -void paravirt_tlb_remove_table(struct mmu_gather *tlb, void *table) -{ -- tlb_remove_page(tlb, table); +- struct ptdesc *ptdesc = (struct ptdesc *)table; +- +- pagetable_dtor(ptdesc); +- tlb_remove_page(tlb, ptdesc_page(ptdesc)); -} --#endif +-#else +-static inline +-void paravirt_tlb_remove_table(struct mmu_gather *tlb, void *table) +-{ +- tlb_remove_table(tlb, table); +-} +-#endif /* !CONFIG_PT_RECLAIM */ +-#endif /* !CONFIG_PARAVIRT */ - gfp_t __userpte_alloc_gfp = GFP_PGTABLE_USER | PGTABLE_HIGHMEM; pgtable_t pte_alloc_one(struct mm_struct *mm) -@@ -54,7 +46,7 @@ void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte) +@@ -64,7 +45,7 @@ early_param("userpte", setup_userpte); + void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte) { - pagetable_pte_dtor(page_ptdesc(pte)); paravirt_release_pte(page_to_pfn(pte)); -- paravirt_tlb_remove_table(tlb, pte); +- paravirt_tlb_remove_table(tlb, page_ptdesc(pte)); + tlb_remove_table(tlb, pte); } #if CONFIG_PGTABLE_LEVELS > 2 -@@ -70,7 +62,7 @@ void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd) +@@ -78,21 +59,21 @@ void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd) + #ifdef CONFIG_X86_PAE tlb->need_flush_all = 1; #endif - pagetable_pmd_dtor(ptdesc); -- paravirt_tlb_remove_table(tlb, ptdesc_page(ptdesc)); -+ tlb_remove_table(tlb, ptdesc_page(ptdesc)); +- paravirt_tlb_remove_table(tlb, virt_to_ptdesc(pmd)); ++ tlb_remove_table(tlb, virt_to_ptdesc(pmd)); } #if CONFIG_PGTABLE_LEVELS > 3 -@@ -80,14 +72,14 @@ void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud) - - pagetable_pud_dtor(ptdesc); + void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud) + { paravirt_release_pud(__pa(pud) >> PAGE_SHIFT); -- paravirt_tlb_remove_table(tlb, virt_to_page(pud)); +- paravirt_tlb_remove_table(tlb, virt_to_ptdesc(pud)); + tlb_remove_table(tlb, virt_to_page(pud)); } @@ -1641,13 +719,13 @@ index 5745a354a241..3dc4af1f7868 100644 void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d) { paravirt_release_p4d(__pa(p4d) >> PAGE_SHIFT); -- paravirt_tlb_remove_table(tlb, virt_to_page(p4d)); +- paravirt_tlb_remove_table(tlb, virt_to_ptdesc(p4d)); + tlb_remove_table(tlb, virt_to_page(p4d)); } #endif /* CONFIG_PGTABLE_LEVELS > 4 */ #endif /* CONFIG_PGTABLE_LEVELS > 3 */ diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c -index a2becb85bea7..682da8d0d1c9 100644 +index 6cf881a942bb..682da8d0d1c9 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -74,13 +74,15 @@ @@ -1999,36 +1077,21 @@ index a2becb85bea7..682da8d0d1c9 100644 /* * If the CPU is not in lazy TLB mode, we are just switching * from one thread in a process to another thread in the same -@@ -607,30 +907,32 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next, +@@ -606,6 +906,13 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next, + */ cond_mitigation(tsk); - /* -- * Stop remote flushes for the previous mm. -- * Skip kernel threads; we never send init_mm TLB flushing IPIs, -- * but the bitmap manipulation can cause cache line contention. ++ /* + * Let nmi_uaccess_okay() and finish_asid_transition() + * know that we're changing CR3. - */ -- if (prev != &init_mm) { -- VM_WARN_ON_ONCE(!cpumask_test_cpu(cpu, -- mm_cpumask(prev))); -- cpumask_clear_cpu(cpu, mm_cpumask(prev)); -- } ++ */ + this_cpu_write(cpu_tlbstate.loaded_mm, LOADED_MM_SWITCHING); + barrier(); + -+ /* -+ * Leave this CPU in prev's mm_cpumask. Atomic writes to -+ * mm_cpumask can be expensive under contention. The CPU -+ * will be removed lazily at TLB flush time. -+ */ -+ VM_WARN_ON_ONCE(prev != &init_mm && !cpumask_test_cpu(cpu, -+ mm_cpumask(prev))); - - /* Start receiving IPIs and then read tlb_gen (and LAM below) */ -- if (next != &init_mm) -+ if (next != &init_mm && !cpumask_test_cpu(cpu, mm_cpumask(next))) - cpumask_set_cpu(cpu, mm_cpumask(next)); + /* + * Leave this CPU in prev's mm_cpumask. Atomic writes to + * mm_cpumask can be expensive under contention. The CPU +@@ -620,14 +927,12 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next, next_tlb_gen = atomic64_read(&next->context.tlb_gen); choose_new_asid(next, next_tlb_gen, &new_asid, &need_flush); @@ -2045,7 +1108,7 @@ index a2becb85bea7..682da8d0d1c9 100644 this_cpu_write(cpu_tlbstate.ctxs[new_asid].ctx_id, next->context.ctx_id); this_cpu_write(cpu_tlbstate.ctxs[new_asid].tlb_gen, next_tlb_gen); load_new_mm_cr3(next->pgd, new_asid, new_lam, true); -@@ -749,7 +1051,7 @@ static void flush_tlb_func(void *info) +@@ -746,7 +1051,7 @@ static void flush_tlb_func(void *info) const struct flush_tlb_info *f = info; struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm); u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid); @@ -2054,22 +1117,7 @@ index a2becb85bea7..682da8d0d1c9 100644 bool local = smp_processor_id() == f->initiating_cpu; unsigned long nr_invalidate = 0; u64 mm_tlb_gen; -@@ -760,15 +1062,28 @@ static void flush_tlb_func(void *info) - if (!local) { - inc_irq_stat(irq_tlb_count); - count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED); -+ } - -- /* Can only happen on remote CPUs */ -- if (f->mm && f->mm != loaded_mm) -- return; -+ /* The CPU was left in the mm_cpumask of the target mm. Clear it. */ -+ if (f->mm && f->mm != loaded_mm) { -+ cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(f->mm)); -+ trace_tlb_flush(TLB_REMOTE_WRONG_CPU, 0); -+ return; - } - +@@ -769,6 +1074,16 @@ static void flush_tlb_func(void *info) if (unlikely(loaded_mm == &init_mm)) return; @@ -2095,46 +1143,7 @@ index a2becb85bea7..682da8d0d1c9 100644 if (unlikely(f->new_tlb_gen != TLB_GENERATION_INVALID && f->new_tlb_gen <= local_tlb_gen)) { /* -@@ -893,9 +1210,36 @@ static void flush_tlb_func(void *info) - nr_invalidate); - } - --static bool tlb_is_not_lazy(int cpu, void *data) -+static bool should_flush_tlb(int cpu, void *data) - { -- return !per_cpu(cpu_tlbstate_shared.is_lazy, cpu); -+ struct flush_tlb_info *info = data; -+ -+ /* Lazy TLB will get flushed at the next context switch. */ -+ if (per_cpu(cpu_tlbstate_shared.is_lazy, cpu)) -+ return false; -+ -+ /* No mm means kernel memory flush. */ -+ if (!info->mm) -+ return true; -+ -+ /* The target mm is loaded, and the CPU is not lazy. */ -+ if (per_cpu(cpu_tlbstate.loaded_mm, cpu) == info->mm) -+ return true; -+ -+ /* In cpumask, but not the loaded mm? Periodically remove by flushing. */ -+ if (info->trim_cpumask) -+ return true; -+ -+ return false; -+} -+ -+static bool should_trim_cpumask(struct mm_struct *mm) -+{ -+ if (time_after(jiffies, READ_ONCE(mm->context.next_trim_cpumask))) { -+ WRITE_ONCE(mm->context.next_trim_cpumask, jiffies + HZ); -+ return true; -+ } -+ return false; - } - - DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state_shared, cpu_tlbstate_shared); -@@ -926,10 +1270,10 @@ STATIC_NOPV void native_flush_tlb_multi(const struct cpumask *cpumask, +@@ -953,7 +1270,7 @@ STATIC_NOPV void native_flush_tlb_multi(const struct cpumask *cpumask, * up on the new contents of what used to be page tables, while * doing a speculative memory access. */ @@ -2142,17 +1151,11 @@ index a2becb85bea7..682da8d0d1c9 100644 + if (info->freed_tables || in_asid_transition(info)) on_each_cpu_mask(cpumask, flush_tlb_func, (void *)info, true); else -- on_each_cpu_cond_mask(tlb_is_not_lazy, flush_tlb_func, -+ on_each_cpu_cond_mask(should_flush_tlb, flush_tlb_func, - (void *)info, 1, cpumask); - } - -@@ -980,6 +1324,16 @@ static struct flush_tlb_info *get_flush_tlb_info(struct mm_struct *mm, - info->freed_tables = freed_tables; - info->new_tlb_gen = new_tlb_gen; + on_each_cpu_cond_mask(should_flush_tlb, flush_tlb_func, +@@ -1009,6 +1326,15 @@ static struct flush_tlb_info *get_flush_tlb_info(struct mm_struct *mm, info->initiating_cpu = smp_processor_id(); -+ info->trim_cpumask = 0; -+ + info->trim_cpumask = 0; + + /* + * If the number of flushes is so large that a full flush + * would be faster, do a full flush. @@ -2161,10 +1164,11 @@ index a2becb85bea7..682da8d0d1c9 100644 + info->start = 0; + info->end = TLB_FLUSH_ALL; + } - ++ return info; } -@@ -998,17 +1352,8 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, + +@@ -1026,17 +1352,8 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, bool freed_tables) { struct flush_tlb_info *info; @@ -2183,7 +1187,7 @@ index a2becb85bea7..682da8d0d1c9 100644 /* This is also a barrier that synchronizes with switch_mm(). */ new_tlb_gen = inc_mm_tlb_gen(mm); -@@ -1021,8 +1366,12 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, +@@ -1049,9 +1366,12 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, * a local TLB flush is needed. Optimize this use-case by calling * flush_tlb_func_local() directly in this case. */ @@ -2191,13 +1195,13 @@ index a2becb85bea7..682da8d0d1c9 100644 + if (mm_global_asid(mm)) { + broadcast_tlb_flush(info); + } else if (cpumask_any_but(mm_cpumask(mm), cpu) < nr_cpu_ids) { -+ info->trim_cpumask = should_trim_cpumask(mm); + info->trim_cpumask = should_trim_cpumask(mm); flush_tlb_multi(mm_cpumask(mm), info); + consider_global_asid(mm); } else if (mm == this_cpu_read(cpu_tlbstate.loaded_mm)) { lockdep_assert_irqs_enabled(); local_irq_disable(); -@@ -1036,6 +1385,19 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, +@@ -1065,6 +1385,19 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, } @@ -2217,7 +1221,7 @@ index a2becb85bea7..682da8d0d1c9 100644 static void do_flush_tlb_all(void *info) { count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED); -@@ -1044,10 +1406,36 @@ static void do_flush_tlb_all(void *info) +@@ -1073,10 +1406,36 @@ static void do_flush_tlb_all(void *info) void flush_tlb_all(void) { @@ -2254,7 +1258,7 @@ index a2becb85bea7..682da8d0d1c9 100644 static void do_kernel_range_flush(void *info) { struct flush_tlb_info *f = info; -@@ -1060,22 +1448,21 @@ static void do_kernel_range_flush(void *info) +@@ -1089,22 +1448,21 @@ static void do_kernel_range_flush(void *info) void flush_tlb_kernel_range(unsigned long start, unsigned long end) { @@ -2265,12 +1269,12 @@ index a2becb85bea7..682da8d0d1c9 100644 - } else { - struct flush_tlb_info *info; + struct flush_tlb_info *info; ++ ++ guard(preempt)(); - preempt_disable(); - info = get_flush_tlb_info(NULL, start, end, 0, false, - TLB_GENERATION_INVALID); -+ guard(preempt)(); -+ + info = get_flush_tlb_info(NULL, start, end, PAGE_SHIFT, false, + TLB_GENERATION_INVALID); @@ -2288,7 +1292,7 @@ index a2becb85bea7..682da8d0d1c9 100644 } /* -@@ -1247,7 +1634,7 @@ void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) +@@ -1276,7 +1634,7 @@ void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) int cpu = get_cpu(); @@ -2297,7 +1301,7 @@ index a2becb85bea7..682da8d0d1c9 100644 TLB_GENERATION_INVALID); /* * flush_tlb_multi() is not optimized for the common case in which only -@@ -1263,12 +1650,65 @@ void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) +@@ -1292,12 +1650,65 @@ void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) local_irq_enable(); } @@ -2364,10 +1368,10 @@ index a2becb85bea7..682da8d0d1c9 100644 * Blindly accessing user memory from NMI context can be dangerous * if we're in the middle of switching the current user task or diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c -index 55a4996d0c04..041e17282af0 100644 +index 2c70cd35e72c..a0b371557125 100644 --- a/arch/x86/xen/mmu_pv.c +++ b/arch/x86/xen/mmu_pv.c -@@ -2137,7 +2137,6 @@ static const typeof(pv_ops) xen_mmu_ops __initconst = { +@@ -2141,7 +2141,6 @@ static const typeof(pv_ops) xen_mmu_ops __initconst = { .flush_tlb_kernel = xen_flush_tlb, .flush_tlb_one_user = xen_flush_tlb_one_user, .flush_tlb_multi = xen_flush_tlb_multi, @@ -2375,82 +1379,6 @@ index 55a4996d0c04..041e17282af0 100644 .pgd_alloc = xen_pgd_alloc, .pgd_free = xen_pgd_free, -diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h -index 332cee285662..29e6d8e6d0e5 100644 ---- a/include/linux/mm_types.h -+++ b/include/linux/mm_types.h -@@ -1401,6 +1401,7 @@ enum tlb_flush_reason { - TLB_LOCAL_SHOOTDOWN, - TLB_LOCAL_MM_SHOOTDOWN, - TLB_REMOTE_SEND_IPI, -+ TLB_REMOTE_WRONG_CPU, - NR_TLB_FLUSH_REASONS, - }; - -diff --git a/mm/memory.c b/mm/memory.c -index 398c031be9ba..3d98aaf9b939 100644 ---- a/mm/memory.c -+++ b/mm/memory.c -@@ -1935,7 +1935,6 @@ void zap_page_range_single(struct vm_area_struct *vma, unsigned long address, - struct mmu_notifier_range range; - struct mmu_gather tlb; - -- lru_add_drain(); - mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, - address, end); - hugetlb_zap_begin(vma, &range.start, &range.end); -diff --git a/mm/mmap.c b/mm/mmap.c -index aec208f90337..d628b7900d2d 100644 ---- a/mm/mmap.c -+++ b/mm/mmap.c -@@ -1664,7 +1664,6 @@ void exit_mmap(struct mm_struct *mm) - goto destroy; - } - -- lru_add_drain(); - flush_cache_mm(mm); - tlb_gather_mmu_fullmm(&tlb, mm); - /* update_hiwater_rss(mm) here? but nobody should be looking */ -@@ -2107,7 +2106,6 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) - vma, new_start, length, false, true)) - return -ENOMEM; - -- lru_add_drain(); - tlb_gather_mmu(&tlb, mm); - next = vma_next(&vmi); - if (new_end > old_start) { -diff --git a/mm/swap_state.c b/mm/swap_state.c -index e0c0321b8ff7..ca42b2be64d9 100644 ---- a/mm/swap_state.c -+++ b/mm/swap_state.c -@@ -317,7 +317,6 @@ void free_pages_and_swap_cache(struct encoded_page **pages, int nr) - struct folio_batch folios; - unsigned int refs[PAGEVEC_SIZE]; - -- lru_add_drain(); - folio_batch_init(&folios); - for (int i = 0; i < nr; i++) { - struct folio *folio = page_folio(encoded_page_ptr(pages[i])); -diff --git a/mm/vma.c b/mm/vma.c -index bb2119e5a0d0..a593d5edfd88 100644 ---- a/mm/vma.c -+++ b/mm/vma.c -@@ -398,7 +398,6 @@ void unmap_region(struct ma_state *mas, struct vm_area_struct *vma, - struct mm_struct *mm = vma->vm_mm; - struct mmu_gather tlb; - -- lru_add_drain(); - tlb_gather_mmu(&tlb, mm); - update_hiwater_rss(mm); - unmap_vmas(&tlb, mas, vma, vma->vm_start, vma->vm_end, vma->vm_end, -@@ -1130,7 +1129,6 @@ static inline void vms_clear_ptes(struct vma_munmap_struct *vms, - * were isolated before we downgraded mmap_lock. - */ - mas_set(mas_detach, 1); -- lru_add_drain(); - tlb_gather_mmu(&tlb, vms->vma->vm_mm); - update_hiwater_rss(vms->vma->vm_mm); - unmap_vmas(&tlb, mas_detach, vms->vma, vms->start, vms->end, diff --git a/tools/arch/x86/include/asm/msr-index.h b/tools/arch/x86/include/asm/msr-index.h index 3ae84c3b8e6d..dc1c1057f26e 100644 --- a/tools/arch/x86/include/asm/msr-index.h @@ -2472,14 +1400,14 @@ index 3ae84c3b8e6d..dc1c1057f26e 100644 /* -- -2.48.0.rc1 +2.48.1 -From 3cfc3cd67b727b62206d7c97efd5ebf120e47555 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Sun, 2 Feb 2025 13:53:27 +0100 -Subject: [PATCH 03/12] bbr3 +From bb9024da7b3d8a2203bf5c444c461a951519025f Mon Sep 17 00:00:00 2001 +From: Eric Naim +Date: Mon, 3 Feb 2025 11:04:53 +0800 +Subject: [PATCH 3/6] bbr3 -Signed-off-by: Peter Jung +Signed-off-by: Eric Naim --- include/linux/tcp.h | 4 +- include/net/inet_connection_sock.h | 4 +- @@ -2530,7 +1458,7 @@ index c7f42844c79a..170250145598 100644 #define ICSK_TIME_RETRANS 1 /* Retransmit timer */ diff --git a/include/net/tcp.h b/include/net/tcp.h -index e9b37b76e894..419fda8c64e5 100644 +index 5b2b04835688..698954920149 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -375,6 +375,8 @@ static inline void tcp_dec_quickack_mode(struct sock *sk) @@ -2736,10 +1664,10 @@ index 86bb2e8b17c9..9d9a3eb2ce9b 100644 union tcp_cc_info { diff --git a/include/uapi/linux/rtnetlink.h b/include/uapi/linux/rtnetlink.h -index db7254d52d93..38de18d921ea 100644 +index 66c3903d29cf..dfdbc1c0b606 100644 --- a/include/uapi/linux/rtnetlink.h +++ b/include/uapi/linux/rtnetlink.h -@@ -507,12 +507,14 @@ enum { +@@ -516,12 +516,14 @@ enum { #define RTAX_FEATURE_TIMESTAMP (1 << 2) /* unused */ #define RTAX_FEATURE_ALLFRAG (1 << 3) /* unused */ #define RTAX_FEATURE_TCP_USEC_TS (1 << 4) @@ -5508,7 +4436,7 @@ index df758adbb445..e98e5dbc050e 100644 icsk->icsk_ca_ops->init(sk); if (tcp_ca_needs_ecn(sk)) diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c -index 4811727b8a02..ba8b714fb693 100644 +index eb82e01da911..5dd9d8370ccb 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -370,7 +370,7 @@ static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb) @@ -5624,7 +4552,7 @@ index 4811727b8a02..ba8b714fb693 100644 return 1; old_ack: -@@ -5752,13 +5777,14 @@ static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible) +@@ -5758,13 +5783,14 @@ static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible) /* More than one full frame received... */ if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss && @@ -5642,10 +4570,10 @@ index 4811727b8a02..ba8b714fb693 100644 tcp_in_quickack_mode(sk) || /* Protocol state mandates a one-time immediate ACK */ diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c -index 7121d8573928..696afe8cfda8 100644 +index b089b08e9617..79e776a4f7f2 100644 --- a/net/ipv4/tcp_minisocks.c +++ b/net/ipv4/tcp_minisocks.c -@@ -466,6 +466,8 @@ void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst) +@@ -471,6 +471,8 @@ void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst) u32 ca_key = dst_metric(dst, RTAX_CC_ALGO); bool ca_got_dst = false; @@ -5655,10 +4583,10 @@ index 7121d8573928..696afe8cfda8 100644 const struct tcp_congestion_ops *ca; diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c -index 0e5b9a654254..f7da9d719b25 100644 +index bc95d2a5924f..d4c45ca6fe06 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c -@@ -336,10 +336,9 @@ static void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) +@@ -339,10 +339,9 @@ static void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) bool bpf_needs_ecn = tcp_bpf_ca_needs_ecn(sk); bool use_ecn = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn) == 1 || tcp_ca_needs_ecn(sk) || bpf_needs_ecn; @@ -5670,7 +4598,7 @@ index 0e5b9a654254..f7da9d719b25 100644 if (dst && dst_feature(dst, RTAX_FEATURE_ECN)) use_ecn = true; } -@@ -351,6 +350,9 @@ static void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) +@@ -354,6 +353,9 @@ static void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) tp->ecn_flags = TCP_ECN_OK; if (tcp_ca_needs_ecn(sk) || bpf_needs_ecn) INET_ECN_xmit(sk); @@ -5680,7 +4608,7 @@ index 0e5b9a654254..f7da9d719b25 100644 } } -@@ -388,7 +390,8 @@ static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb, +@@ -391,7 +393,8 @@ static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb, th->cwr = 1; skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; } @@ -5690,7 +4618,7 @@ index 0e5b9a654254..f7da9d719b25 100644 /* ACK or retransmitted segment: clear ECT|CE */ INET_ECN_dontxmit(sk); } -@@ -1603,7 +1606,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, +@@ -1606,7 +1609,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, { struct tcp_sock *tp = tcp_sk(sk); struct sk_buff *buff; @@ -5699,7 +4627,7 @@ index 0e5b9a654254..f7da9d719b25 100644 long limit; int nlen; u8 flags; -@@ -1678,6 +1681,30 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, +@@ -1681,6 +1684,30 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, if (diff) tcp_adjust_pcount(sk, skb, diff); @@ -5730,7 +4658,7 @@ index 0e5b9a654254..f7da9d719b25 100644 } /* Link BUFF into the send queue. */ -@@ -2035,13 +2062,12 @@ static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, +@@ -2038,13 +2065,12 @@ static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now) { const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; @@ -5749,7 +4677,7 @@ index 0e5b9a654254..f7da9d719b25 100644 return min_t(u32, tso_segs, sk->sk_gso_max_segs); } -@@ -2767,6 +2793,7 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, +@@ -2770,6 +2796,7 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, skb_set_delivery_time(skb, tp->tcp_wstamp_ns, SKB_CLOCK_MONOTONIC); list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue); tcp_init_tso_segs(skb, mss_now); @@ -5757,7 +4685,7 @@ index 0e5b9a654254..f7da9d719b25 100644 goto repair; /* Skip network transmission */ } -@@ -2979,6 +3006,7 @@ void tcp_send_loss_probe(struct sock *sk) +@@ -2982,6 +3009,7 @@ void tcp_send_loss_probe(struct sock *sk) if (WARN_ON(!skb || !tcp_skb_pcount(skb))) goto rearm_timer; @@ -5858,17 +4786,16 @@ index b412ed88ccd9..d70f8b742b21 100644 event = icsk->icsk_pending; -- -2.48.0.rc1 +2.48.1 -From a3411065e4db0cdb3565bf9daf18b2ba8fe3b384 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Sun, 2 Feb 2025 13:53:37 +0100 -Subject: [PATCH 04/12] cachy +From 84eaf13f0d53531674f646cefcbb29a597edb48a Mon Sep 17 00:00:00 2001 +From: Eric Naim +Date: Mon, 3 Feb 2025 12:01:15 +0800 +Subject: [PATCH 4/6] cachy -Signed-off-by: Peter Jung +Signed-off-by: Eric Naim --- .../admin-guide/kernel-parameters.txt | 12 + - Documentation/admin-guide/sysctl/vm.rst | 72 + Makefile | 8 + arch/x86/Kconfig.cpu | 367 +- arch/x86/Makefile | 89 +- @@ -5914,8 +4841,7 @@ Signed-off-by: Peter Jung drivers/scsi/Makefile | 1 + drivers/scsi/vhba/Kconfig | 9 + drivers/scsi/vhba/Makefile | 4 + - drivers/scsi/vhba/vhba.c | 1130 ++++++ - include/linux/mm.h | 8 + + drivers/scsi/vhba/vhba.c | 1132 ++++++ include/linux/pagemap.h | 2 +- include/linux/user_namespace.h | 4 + include/linux/wait.h | 2 + @@ -5927,19 +4853,18 @@ Signed-off-by: Peter Jung kernel/sched/fair.c | 13 + kernel/sched/sched.h | 2 +- kernel/sched/wait.c | 24 + - kernel/sysctl.c | 46 + + kernel/sysctl.c | 12 + kernel/user_namespace.c | 7 + - mm/Kconfig | 65 +- + mm/Kconfig | 2 +- mm/compaction.c | 4 + mm/huge_memory.c | 4 + - mm/mm_init.c | 1 + mm/page-writeback.c | 8 + mm/page_alloc.c | 4 + mm/swap.c | 5 + mm/vmpressure.c | 4 + - mm/vmscan.c | 143 + + mm/vmscan.c | 4 + net/ipv4/inet_connection_sock.c | 2 +- - 72 files changed, 6714 insertions(+), 93 deletions(-) + 69 files changed, 6399 insertions(+), 93 deletions(-) create mode 100644 drivers/media/v4l2-core/v4l2loopback.c create mode 100644 drivers/media/v4l2-core/v4l2loopback.h create mode 100644 drivers/media/v4l2-core/v4l2loopback_formats.h @@ -5949,10 +4874,10 @@ Signed-off-by: Peter Jung create mode 100644 drivers/scsi/vhba/vhba.c diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt -index 3872bc6ec49d..5e8881ec6b40 100644 +index fb8752b42ec8..753f36fd9361 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt -@@ -2256,6 +2256,9 @@ +@@ -2277,6 +2277,9 @@ disable Do not enable intel_pstate as the default scaling driver for the supported processors @@ -5962,7 +4887,7 @@ index 3872bc6ec49d..5e8881ec6b40 100644 active Use intel_pstate driver to bypass the scaling governors layer of cpufreq and provides it own -@@ -4481,6 +4484,15 @@ +@@ -4642,6 +4645,15 @@ nomsi [MSI] If the PCI_MSI kernel config parameter is enabled, this kernel boot option can be used to disable the use of MSI interrupts system-wide. @@ -5978,108 +4903,11 @@ index 3872bc6ec49d..5e8881ec6b40 100644 noioapicquirk [APIC] Disable all boot interrupt quirks. Safety option to keep boot IRQs enabled. This should never be necessary. -diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst -index f48eaa98d22d..fc777c14cff6 100644 ---- a/Documentation/admin-guide/sysctl/vm.rst -+++ b/Documentation/admin-guide/sysctl/vm.rst -@@ -25,6 +25,9 @@ files can be found in mm/swap.c. - Currently, these files are in /proc/sys/vm: - - - admin_reserve_kbytes -+- anon_min_ratio -+- clean_low_ratio -+- clean_min_ratio - - compact_memory - - compaction_proactiveness - - compact_unevictable_allowed -@@ -108,6 +111,67 @@ On x86_64 this is about 128MB. - Changing this takes effect whenever an application requests memory. - - -+anon_min_ratio -+============== -+ -+This knob provides *hard* protection of anonymous pages. The anonymous pages -+on the current node won't be reclaimed under any conditions when their amount -+is below vm.anon_min_ratio. -+ -+This knob may be used to prevent excessive swap thrashing when anonymous -+memory is low (for example, when memory is going to be overfilled by -+compressed data of zram module). -+ -+Setting this value too high (close to 100) can result in inability to -+swap and can lead to early OOM under memory pressure. -+ -+The unit of measurement is the percentage of the total memory of the node. -+ -+The default value is 15. -+ -+ -+clean_low_ratio -+================ -+ -+This knob provides *best-effort* protection of clean file pages. The file pages -+on the current node won't be reclaimed under memory pressure when the amount of -+clean file pages is below vm.clean_low_ratio *unless* we threaten to OOM. -+ -+Protection of clean file pages using this knob may be used when swapping is -+still possible to -+ - prevent disk I/O thrashing under memory pressure; -+ - improve performance in disk cache-bound tasks under memory pressure. -+ -+Setting it to a high value may result in a early eviction of anonymous pages -+into the swap space by attempting to hold the protected amount of clean file -+pages in memory. -+ -+The unit of measurement is the percentage of the total memory of the node. -+ -+The default value is 0. -+ -+ -+clean_min_ratio -+================ -+ -+This knob provides *hard* protection of clean file pages. The file pages on the -+current node won't be reclaimed under memory pressure when the amount of clean -+file pages is below vm.clean_min_ratio. -+ -+Hard protection of clean file pages using this knob may be used to -+ - prevent disk I/O thrashing under memory pressure even with no free swap space; -+ - improve performance in disk cache-bound tasks under memory pressure; -+ - avoid high latency and prevent livelock in near-OOM conditions. -+ -+Setting it to a high value may result in a early out-of-memory condition due to -+the inability to reclaim the protected amount of clean file pages when other -+types of pages cannot be reclaimed. -+ -+The unit of measurement is the percentage of the total memory of the node. -+ -+The default value is 15. -+ -+ - compact_memory - ============== - -@@ -964,6 +1028,14 @@ be 133 (x + 2x = 200, 2x = 133.33). - At 0, the kernel will not initiate swap until the amount of free and - file-backed pages is less than the high watermark in a zone. - -+This knob has no effect if the amount of clean file pages on the current -+node is below vm.clean_low_ratio or vm.clean_min_ratio. In this case, -+only anonymous pages can be reclaimed. -+ -+If the number of anonymous pages on the current node is below -+vm.anon_min_ratio, then only file pages can be reclaimed with -+any vm.swappiness value. -+ - - unprivileged_userfaultfd - ======================== diff --git a/Makefile b/Makefile -index 7bc322bc7ad8..aba8664829c0 100644 +index 9e0d63d9d94b..aeb4211d6b64 100644 --- a/Makefile +++ b/Makefile -@@ -860,11 +860,19 @@ KBUILD_CFLAGS += -fno-delete-null-pointer-checks +@@ -861,11 +861,19 @@ KBUILD_CFLAGS += -fno-delete-null-pointer-checks ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE KBUILD_CFLAGS += -O2 KBUILD_RUSTFLAGS += -Copt-level=2 @@ -6788,10 +5616,10 @@ index ddb798603201..7c20387d8202 100644 } -#endif diff --git a/block/elevator.c b/block/elevator.c -index 7c3ba80e5ff4..06e974eb6594 100644 +index cd2ce4921601..bcd6d2482a8a 100644 --- a/block/elevator.c +++ b/block/elevator.c -@@ -566,9 +566,17 @@ static struct elevator_type *elevator_get_default(struct request_queue *q) +@@ -558,9 +558,17 @@ static struct elevator_type *elevator_get_default(struct request_queue *q) if (q->nr_hw_queues != 1 && !blk_mq_is_shared_tags(q->tag_set->flags)) @@ -6843,7 +5671,7 @@ index 45d1c3e630f7..4f5ab2429a7f 100644 obj-$(CONFIG_MTD) += mtd/ obj-$(CONFIG_SPI) += spi/ diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c -index 8d27c567be1c..479477438de0 100644 +index f813dbdc2346..86af3e5190e0 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -1618,7 +1618,7 @@ static irqreturn_t ahci_thunderx_irq_handler(int irq, void *dev_instance) @@ -6919,10 +5747,10 @@ index 97c2d4f15d76..5a3af44d785a 100644 This driver adds a CPUFreq driver which utilizes a fine grain processor performance frequency control range instead of legacy diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c -index b8e2396a708a..d8e529cd454d 100644 +index 9c4cc01fd51a..06a3c53e116e 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c -@@ -3819,6 +3819,8 @@ static int __init intel_pstate_setup(char *str) +@@ -3827,6 +3827,8 @@ static int __init intel_pstate_setup(char *str) if (!strcmp(str, "disable")) no_load = 1; @@ -6932,7 +5760,7 @@ index b8e2396a708a..d8e529cd454d 100644 default_driver = &intel_pstate; else if (!strcmp(str, "passive")) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h -index 4653a8d2823a..6590e83dfbf0 100644 +index 69895fccb474..5884de0f23bb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -160,6 +160,7 @@ struct amdgpu_watchdog_timer { @@ -7032,10 +5860,10 @@ index 0e16432d9a72..867bc5c5ce67 100644 bool amdgpu_atombios_has_dce_engine_info(struct amdgpu_device *adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c -index cd4fac120834..1ab433d774cc 100644 +index d100bb7a137c..4a9580fb0628 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c -@@ -4461,8 +4461,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, +@@ -4465,8 +4465,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, goto failed; } /* init i2c buses */ @@ -7045,7 +5873,7 @@ index cd4fac120834..1ab433d774cc 100644 } } -@@ -4724,8 +4723,7 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev) +@@ -4735,8 +4734,7 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev) amdgpu_reset_fini(adev); /* free i2c buses */ @@ -7056,7 +5884,7 @@ index cd4fac120834..1ab433d774cc 100644 if (amdgpu_emu_mode != 1) amdgpu_atombios_fini(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c -index 38686203bea6..811d020f3f4b 100644 +index 817116e53d44..2b4a4eed2ae4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -137,6 +137,7 @@ enum AMDGPU_DEBUG_MASK { @@ -7138,10 +5966,10 @@ index 5e3faefc5510..6da4f946cac0 100644 #define TO_DM_AUX(x) container_of((x), struct amdgpu_dm_dp_aux, aux) diff --git a/drivers/gpu/drm/amd/display/Kconfig b/drivers/gpu/drm/amd/display/Kconfig -index 11e3f2f3b174..7b1bd69dc29e 100644 +index abd3b6564373..46937e6fa78d 100644 --- a/drivers/gpu/drm/amd/display/Kconfig +++ b/drivers/gpu/drm/amd/display/Kconfig -@@ -54,4 +54,10 @@ config DRM_AMD_SECURE_DISPLAY +@@ -56,4 +56,10 @@ config DRM_AMD_SECURE_DISPLAY This option enables the calculation of crc of specific region via debugfs. Cooperate with specific DMCU FW. @@ -7153,10 +5981,10 @@ index 11e3f2f3b174..7b1bd69dc29e 100644 + endmenu diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c -index 5f216d626cbb..382af92c4ff1 100644 +index ac3fd81fecef..0031e1ebf9cd 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c -@@ -177,6 +177,8 @@ static int amdgpu_dm_init(struct amdgpu_device *adev); +@@ -179,6 +179,8 @@ static int amdgpu_dm_init(struct amdgpu_device *adev); static void amdgpu_dm_fini(struct amdgpu_device *adev); static bool is_freesync_video_mode(const struct drm_display_mode *mode, struct amdgpu_dm_connector *aconnector); static void reset_freesync_config_for_crtc(struct dm_crtc_state *new_crtc_state); @@ -7165,7 +5993,7 @@ index 5f216d626cbb..382af92c4ff1 100644 static enum drm_mode_subconnector get_subconnector_type(struct dc_link *link) { -@@ -2839,6 +2841,33 @@ static int amdgpu_dm_smu_write_watermarks_table(struct amdgpu_device *adev) +@@ -2893,6 +2895,33 @@ static int amdgpu_dm_smu_write_watermarks_table(struct amdgpu_device *adev) return 0; } @@ -7199,7 +6027,7 @@ index 5f216d626cbb..382af92c4ff1 100644 /** * dm_hw_init() - Initialize DC device * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. -@@ -2870,6 +2899,10 @@ static int dm_hw_init(struct amdgpu_ip_block *ip_block) +@@ -2924,6 +2953,10 @@ static int dm_hw_init(struct amdgpu_ip_block *ip_block) return r; amdgpu_dm_hpd_init(adev); @@ -7210,7 +6038,7 @@ index 5f216d626cbb..382af92c4ff1 100644 return 0; } -@@ -2885,6 +2918,8 @@ static int dm_hw_fini(struct amdgpu_ip_block *ip_block) +@@ -2939,6 +2972,8 @@ static int dm_hw_fini(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; @@ -7219,7 +6047,7 @@ index 5f216d626cbb..382af92c4ff1 100644 amdgpu_dm_hpd_fini(adev); amdgpu_dm_irq_fini(adev); -@@ -4516,7 +4551,7 @@ static int amdgpu_dm_mode_config_init(struct amdgpu_device *adev) +@@ -4582,7 +4617,7 @@ static int amdgpu_dm_mode_config_init(struct amdgpu_device *adev) return r; } @@ -7228,7 +6056,7 @@ index 5f216d626cbb..382af92c4ff1 100644 if (amdgpu_dm_create_color_properties(adev)) { dc_state_release(state->context); kfree(state); -@@ -8218,7 +8253,7 @@ static int amdgpu_dm_i2c_xfer(struct i2c_adapter *i2c_adap, +@@ -8282,7 +8317,7 @@ static int amdgpu_dm_i2c_xfer(struct i2c_adapter *i2c_adap, int i; int result = -EIO; @@ -7237,7 +6065,7 @@ index 5f216d626cbb..382af92c4ff1 100644 return result; cmd.payloads = kcalloc(num, sizeof(struct i2c_payload), GFP_KERNEL); -@@ -8237,11 +8272,18 @@ static int amdgpu_dm_i2c_xfer(struct i2c_adapter *i2c_adap, +@@ -8301,11 +8336,18 @@ static int amdgpu_dm_i2c_xfer(struct i2c_adapter *i2c_adap, cmd.payloads[i].data = msgs[i].buf; } @@ -7261,7 +6089,7 @@ index 5f216d626cbb..382af92c4ff1 100644 kfree(cmd.payloads); return result; -@@ -8258,9 +8300,7 @@ static const struct i2c_algorithm amdgpu_dm_i2c_algo = { +@@ -8322,9 +8364,7 @@ static const struct i2c_algorithm amdgpu_dm_i2c_algo = { }; static struct amdgpu_i2c_adapter * @@ -7272,7 +6100,7 @@ index 5f216d626cbb..382af92c4ff1 100644 { struct amdgpu_device *adev = ddc_service->ctx->driver_context; struct amdgpu_i2c_adapter *i2c; -@@ -8271,9 +8311,14 @@ create_i2c(struct ddc_service *ddc_service, +@@ -8335,9 +8375,14 @@ create_i2c(struct ddc_service *ddc_service, i2c->base.owner = THIS_MODULE; i2c->base.dev.parent = &adev->pdev->dev; i2c->base.algo = &amdgpu_dm_i2c_algo; @@ -7288,7 +6116,7 @@ index 5f216d626cbb..382af92c4ff1 100644 return i2c; } -@@ -8298,7 +8343,7 @@ static int amdgpu_dm_connector_init(struct amdgpu_display_manager *dm, +@@ -8383,7 +8428,7 @@ static int amdgpu_dm_connector_init(struct amdgpu_display_manager *dm, link->priv = aconnector; @@ -7298,7 +6126,7 @@ index 5f216d626cbb..382af92c4ff1 100644 DRM_ERROR("Failed to create i2c adapter data\n"); return -ENOMEM; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h -index 2227cd8e4a89..5710776bb0e2 100644 +index d2703ca7dff3..ef60e80de19c 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h @@ -606,6 +606,13 @@ struct amdgpu_display_manager { @@ -7360,10 +6188,10 @@ index 36a830a7440f..a8fc8bd52d51 100644 #endif return 0; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c -index 495e3cd70426..704a48209657 100644 +index 774cc3f4f3fd..d5066e407ad5 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c -@@ -1573,7 +1573,7 @@ static void amdgpu_dm_plane_drm_plane_destroy_state(struct drm_plane *plane, +@@ -1595,7 +1595,7 @@ static void amdgpu_dm_plane_drm_plane_destroy_state(struct drm_plane *plane, drm_atomic_helper_plane_destroy_state(plane, state); } @@ -7372,7 +6200,7 @@ index 495e3cd70426..704a48209657 100644 static void dm_atomic_plane_attach_color_mgmt_properties(struct amdgpu_display_manager *dm, struct drm_plane *plane) -@@ -1764,7 +1764,7 @@ static const struct drm_plane_funcs dm_plane_funcs = { +@@ -1786,7 +1786,7 @@ static const struct drm_plane_funcs dm_plane_funcs = { .atomic_duplicate_state = amdgpu_dm_plane_drm_plane_duplicate_state, .atomic_destroy_state = amdgpu_dm_plane_drm_plane_destroy_state, .format_mod_supported = amdgpu_dm_plane_format_mod_supported, @@ -7381,9 +6209,9 @@ index 495e3cd70426..704a48209657 100644 .atomic_set_property = dm_atomic_plane_set_property, .atomic_get_property = dm_atomic_plane_get_property, #endif -@@ -1857,7 +1857,7 @@ int amdgpu_dm_plane_init(struct amdgpu_display_manager *dm, - - drm_plane_helper_add(plane, &dm_plane_helper_funcs); +@@ -1882,7 +1882,7 @@ int amdgpu_dm_plane_init(struct amdgpu_display_manager *dm, + else + drm_plane_helper_add(plane, &dm_plane_helper_funcs); -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR @@ -7391,7 +6219,7 @@ index 495e3cd70426..704a48209657 100644 #endif /* Create (reset) the plane state */ diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c -index c9a6de110b74..470ec970217b 100644 +index a62f6c51301c..1d2c6019efac 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c +++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c @@ -1778,6 +1778,7 @@ static enum bp_result get_firmware_info_v3_1( @@ -7429,10 +6257,10 @@ index c9a6de110b74..470ec970217b 100644 return BP_RESULT_OK; } diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c -index 457d60eeb486..13636eb4ec3f 100644 +index c1b79b379447..261c3bc4d46e 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c -@@ -142,6 +142,12 @@ bool dc_link_update_dsc_config(struct pipe_ctx *pipe_ctx) +@@ -150,6 +150,12 @@ bool dc_link_update_dsc_config(struct pipe_ctx *pipe_ctx) return link->dc->link_srv->update_dsc_config(pipe_ctx); } @@ -7446,10 +6274,10 @@ index 457d60eeb486..13636eb4ec3f 100644 struct dc *dc, size_t slave_address) diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h -index 08c5a315b3a6..70d6005ecd64 100644 +index 053481ab69ef..0c2c0fc45ae5 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h -@@ -1939,6 +1939,9 @@ int dc_link_aux_transfer_raw(struct ddc_service *ddc, +@@ -1947,6 +1947,9 @@ int dc_link_aux_transfer_raw(struct ddc_service *ddc, struct aux_payload *payload, enum aux_return_code_type *operation_result); @@ -7523,10 +6351,10 @@ index e8ae7681bf0a..8a0d873983f3 100644 } diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c -index 21bd635bcdfc..6f4032038fc7 100644 +index 8ca793c222ff..8a06a6c35a4b 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c -@@ -2809,7 +2809,10 @@ int smu_get_power_limit(void *handle, +@@ -2822,7 +2822,10 @@ int smu_get_power_limit(void *handle, *limit = smu->max_power_limit; break; case SMU_PPT_LIMIT_MIN: @@ -7538,7 +6366,7 @@ index 21bd635bcdfc..6f4032038fc7 100644 break; default: return -EINVAL; -@@ -2833,7 +2836,14 @@ static int smu_set_power_limit(void *handle, uint32_t limit) +@@ -2846,7 +2849,14 @@ static int smu_set_power_limit(void *handle, uint32_t limit) if (smu->ppt_funcs->set_power_limit) return smu->ppt_funcs->set_power_limit(smu, limit_type, limit); @@ -7619,10 +6447,10 @@ index b5cbb57ee5f6..a0f7fa1518c6 100644 } diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c -index 1ae2c71bb383..784829ada178 100644 +index 02a2919f4e5a..67fde4125238 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c -@@ -3315,6 +3315,11 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) +@@ -3305,6 +3305,11 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) goto bad; } @@ -11891,10 +10719,10 @@ index 000000000000..e105e6f5cc91 +MODULE_AUTHOR("Daniel Drake "); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c -index 76f4df75b08a..49c1a91c611d 100644 +index b84ff7bade82..bcf5940d0698 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c -@@ -3746,6 +3746,106 @@ static void quirk_no_bus_reset(struct pci_dev *dev) +@@ -3747,6 +3747,106 @@ static void quirk_no_bus_reset(struct pci_dev *dev) dev->dev_flags |= PCI_DEV_FLAGS_NO_BUS_RESET; } @@ -12001,7 +10829,7 @@ index 76f4df75b08a..49c1a91c611d 100644 /* * Some NVIDIA GPU devices do not work with bus reset, SBR needs to be * prevented for those affected devices. -@@ -5170,6 +5270,7 @@ static const struct pci_dev_acs_enabled { +@@ -5171,6 +5271,7 @@ static const struct pci_dev_acs_enabled { { PCI_VENDOR_ID_ZHAOXIN, PCI_ANY_ID, pci_quirk_zhaoxin_pcie_ports_acs }, /* Wangxun nics */ { PCI_VENDOR_ID_WANGXUN, PCI_ANY_ID, pci_quirk_wangxun_nic_acs }, @@ -12059,10 +10887,10 @@ index 000000000000..2d7524b66199 +ccflags-y := -DVHBA_VERSION=\"$(VHBA_VERSION)\" -Werror diff --git a/drivers/scsi/vhba/vhba.c b/drivers/scsi/vhba/vhba.c new file mode 100644 -index 000000000000..7531223355e5 +index 000000000000..878a3be0ba2b --- /dev/null +++ b/drivers/scsi/vhba/vhba.c -@@ -0,0 +1,1130 @@ +@@ -0,0 +1,1132 @@ +/* + * vhba.c + * @@ -12602,8 +11430,10 @@ index 000000000000..7531223355e5 +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0) + .slave_alloc = vhba_slave_alloc, +#endif -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0) ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0) && LINUX_VERSION_CODE < KERNEL_VERSION(6, 14, 0) + .tag_alloc_policy = BLK_TAG_ALLOC_RR, ++#else ++ .tag_alloc_policy_rr = true, +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) + .use_blk_tags = 1, @@ -13193,30 +12023,11 @@ index 000000000000..7531223355e5 +module_init(vhba_init); +module_exit(vhba_exit); + -diff --git a/include/linux/mm.h b/include/linux/mm.h -index b1c3db9cf355..e38685ece897 100644 ---- a/include/linux/mm.h -+++ b/include/linux/mm.h -@@ -206,6 +206,14 @@ static inline void __mm_zero_struct_page(struct page *page) - - extern int sysctl_max_map_count; - -+extern bool sysctl_workingset_protection; -+extern u8 sysctl_anon_min_ratio; -+extern u8 sysctl_clean_low_ratio; -+extern u8 sysctl_clean_min_ratio; -+int vm_workingset_protection_update_handler( -+ const struct ctl_table *table, int write, -+ void __user *buffer, size_t *lenp, loff_t *ppos); -+ - extern unsigned long sysctl_user_reserve_kbytes; - extern unsigned long sysctl_admin_reserve_kbytes; - diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h -index bcf0865a38ae..e748afb0ce06 100644 +index 47bfc6b1b632..435901dbc742 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h -@@ -1365,7 +1365,7 @@ struct readahead_control { +@@ -1373,7 +1373,7 @@ struct readahead_control { ._index = i, \ } @@ -13268,10 +12079,10 @@ index 6d90ad974408..d04768b01364 100644 void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout); diff --git a/init/Kconfig b/init/Kconfig -index a20e6efd3f0f..9437171030e2 100644 +index d0d021b3fa3b..4b0adb48a40d 100644 --- a/init/Kconfig +++ b/init/Kconfig -@@ -154,6 +154,10 @@ config THREAD_INFO_IN_TASK +@@ -157,6 +157,10 @@ config THREAD_INFO_IN_TASK menu "General setup" @@ -13282,7 +12093,7 @@ index a20e6efd3f0f..9437171030e2 100644 config BROKEN bool -@@ -1310,6 +1314,22 @@ config USER_NS +@@ -1323,6 +1327,22 @@ config USER_NS If unsure, say N. @@ -13305,7 +12116,7 @@ index a20e6efd3f0f..9437171030e2 100644 config PID_NS bool "PID Namespaces" default y -@@ -1452,6 +1472,12 @@ config CC_OPTIMIZE_FOR_PERFORMANCE +@@ -1465,6 +1485,12 @@ config CC_OPTIMIZE_FOR_PERFORMANCE with the "-O2" compiler flag for best performance and most helpful compile-time warnings. @@ -13374,7 +12185,7 @@ index 54ea59ff8fbe..18f87e0dd137 100644 help This option turns the kernel into a real-time kernel by replacing diff --git a/kernel/fork.c b/kernel/fork.c -index 9b301180fd41..0cb5431b4d7e 100644 +index 735405a9c5f3..d9658b516bcb 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -106,6 +106,10 @@ @@ -13388,7 +12199,7 @@ index 9b301180fd41..0cb5431b4d7e 100644 #include #include #include -@@ -2158,6 +2162,10 @@ __latent_entropy struct task_struct *copy_process( +@@ -2167,6 +2171,10 @@ __latent_entropy struct task_struct *copy_process( if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS)) return ERR_PTR(-EINVAL); @@ -13399,7 +12210,7 @@ index 9b301180fd41..0cb5431b4d7e 100644 /* * Thread groups must share signals as well, and detached threads * can only be started up within the thread group. -@@ -3311,6 +3319,12 @@ int ksys_unshare(unsigned long unshare_flags) +@@ -3320,6 +3328,12 @@ int ksys_unshare(unsigned long unshare_flags) if (unshare_flags & CLONE_NEWNS) unshare_flags |= CLONE_FS; @@ -13435,10 +12246,10 @@ index 2ddb827e3bea..464049c4af3f 100644 return state; diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c -index 26958431deb7..8c0f17a96d4f 100644 +index f9464644186c..457fb08efc66 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c -@@ -73,10 +73,19 @@ unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; +@@ -76,10 +76,19 @@ unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; * * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds) */ @@ -13458,7 +12269,7 @@ index 26958431deb7..8c0f17a96d4f 100644 static int __init setup_sched_thermal_decay_shift(char *str) { -@@ -121,8 +130,12 @@ int __weak arch_asym_cpu_priority(int cpu) +@@ -124,8 +133,12 @@ int __weak arch_asym_cpu_priority(int cpu) * * (default: 5 msec, units: microseconds) */ @@ -13472,10 +12283,10 @@ index 26958431deb7..8c0f17a96d4f 100644 #ifdef CONFIG_NUMA_BALANCING /* Restrict the NUMA promotion throughput (MB/s) for each target node. */ diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h -index c5d67a43fe52..da653eba7884 100644 +index f5cd42d1e90c..ed5a75725411 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h -@@ -2820,7 +2820,7 @@ extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); +@@ -2836,7 +2836,7 @@ extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); extern void wakeup_preempt(struct rq *rq, struct task_struct *p, int flags); @@ -13527,7 +12338,7 @@ index 51e38f5f4701..c5cc616484ba 100644 { wq_entry->flags = flags; diff --git a/kernel/sysctl.c b/kernel/sysctl.c -index 5c9202cb8f59..de4ddf79fe97 100644 +index cb57da499ebb..f7f1c25b30fe 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -80,6 +80,9 @@ @@ -13540,7 +12351,7 @@ index 5c9202cb8f59..de4ddf79fe97 100644 /* shared constants to be used in various sysctls */ const int sysctl_vals[] = { 0, 1, 2, 3, 4, 100, 200, 1000, 3000, INT_MAX, 65535, -1 }; -@@ -1617,6 +1620,15 @@ static struct ctl_table kern_table[] = { +@@ -1617,6 +1620,15 @@ static const struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = proc_dointvec, }, @@ -13556,47 +12367,6 @@ index 5c9202cb8f59..de4ddf79fe97 100644 #ifdef CONFIG_PROC_SYSCTL { .procname = "tainted", -@@ -2197,6 +2209,40 @@ static struct ctl_table vm_table[] = { - .extra1 = SYSCTL_ZERO, - }, - #endif -+ { -+ .procname = "workingset_protection", -+ .data = &sysctl_workingset_protection, -+ .maxlen = sizeof(bool), -+ .mode = 0644, -+ .proc_handler = &proc_dobool, -+ }, -+ { -+ .procname = "anon_min_ratio", -+ .data = &sysctl_anon_min_ratio, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = &vm_workingset_protection_update_handler, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = SYSCTL_ONE_HUNDRED, -+ }, -+ { -+ .procname = "clean_low_ratio", -+ .data = &sysctl_clean_low_ratio, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = &vm_workingset_protection_update_handler, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = SYSCTL_ONE_HUNDRED, -+ }, -+ { -+ .procname = "clean_min_ratio", -+ .data = &sysctl_clean_min_ratio, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = &vm_workingset_protection_update_handler, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = SYSCTL_ONE_HUNDRED, -+ }, - { - .procname = "user_reserve_kbytes", - .data = &sysctl_user_reserve_kbytes, diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c index aa0b2e47f2f2..d74d857b1696 100644 --- a/kernel/user_namespace.c @@ -13616,80 +12386,10 @@ index aa0b2e47f2f2..d74d857b1696 100644 static DEFINE_MUTEX(userns_state_mutex); diff --git a/mm/Kconfig b/mm/Kconfig -index 84000b016808..1d96c5cc35d3 100644 +index 1b501db06417..c5b54d2197aa 100644 --- a/mm/Kconfig +++ b/mm/Kconfig -@@ -499,6 +499,69 @@ config ARCH_WANT_OPTIMIZE_DAX_VMEMMAP - config ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP - bool - -+config ANON_MIN_RATIO -+ int "Default value for vm.anon_min_ratio" -+ depends on SYSCTL -+ range 0 100 -+ default 15 -+ help -+ This option sets the default value for vm.anon_min_ratio sysctl knob. -+ -+ The vm.anon_min_ratio sysctl knob provides *hard* protection of -+ anonymous pages. The anonymous pages on the current node won't be -+ reclaimed under any conditions when their amount is below -+ vm.anon_min_ratio. This knob may be used to prevent excessive swap -+ thrashing when anonymous memory is low (for example, when memory is -+ going to be overfilled by compressed data of zram module). -+ -+ Setting this value too high (close to MemTotal) can result in -+ inability to swap and can lead to early OOM under memory pressure. -+ -+config CLEAN_LOW_RATIO -+ int "Default value for vm.clean_low_ratio" -+ depends on SYSCTL -+ range 0 100 -+ default 0 -+ help -+ This option sets the default value for vm.clean_low_ratio sysctl knob. -+ -+ The vm.clean_low_ratio sysctl knob provides *best-effort* -+ protection of clean file pages. The file pages on the current node -+ won't be reclaimed under memory pressure when the amount of clean file -+ pages is below vm.clean_low_ratio *unless* we threaten to OOM. -+ Protection of clean file pages using this knob may be used when -+ swapping is still possible to -+ - prevent disk I/O thrashing under memory pressure; -+ - improve performance in disk cache-bound tasks under memory -+ pressure. -+ -+ Setting it to a high value may result in a early eviction of anonymous -+ pages into the swap space by attempting to hold the protected amount -+ of clean file pages in memory. -+ -+config CLEAN_MIN_RATIO -+ int "Default value for vm.clean_min_ratio" -+ depends on SYSCTL -+ range 0 100 -+ default 15 -+ help -+ This option sets the default value for vm.clean_min_ratio sysctl knob. -+ -+ The vm.clean_min_ratio sysctl knob provides *hard* protection of -+ clean file pages. The file pages on the current node won't be -+ reclaimed under memory pressure when the amount of clean file pages is -+ below vm.clean_min_ratio. Hard protection of clean file pages using -+ this knob may be used to -+ - prevent disk I/O thrashing under memory pressure even with no free -+ swap space; -+ - improve performance in disk cache-bound tasks under memory -+ pressure; -+ - avoid high latency and prevent livelock in near-OOM conditions. -+ -+ Setting it to a high value may result in a early out-of-memory condition -+ due to the inability to reclaim the protected amount of clean file pages -+ when other types of pages cannot be reclaimed. -+ - config HAVE_MEMBLOCK_PHYS_MAP - bool - -@@ -648,7 +711,7 @@ config COMPACTION +@@ -691,7 +691,7 @@ config COMPACTION config COMPACT_UNEVICTABLE_DEFAULT int depends on COMPACTION @@ -13699,10 +12399,10 @@ index 84000b016808..1d96c5cc35d3 100644 # diff --git a/mm/compaction.c b/mm/compaction.c -index a2b16b08cbbf..48d611e58ad3 100644 +index 12ed8425fa17..da2961bd26b6 100644 --- a/mm/compaction.c +++ b/mm/compaction.c -@@ -1920,7 +1920,11 @@ static int sysctl_compact_unevictable_allowed __read_mostly = CONFIG_COMPACT_UNE +@@ -1923,7 +1923,11 @@ static int sysctl_compact_unevictable_allowed __read_mostly = CONFIG_COMPACT_UNE * aggressively the kernel should compact memory in the * background. It takes values in the range [0, 100]. */ @@ -13715,7 +12415,7 @@ index a2b16b08cbbf..48d611e58ad3 100644 static int __read_mostly sysctl_compact_memory; diff --git a/mm/huge_memory.c b/mm/huge_memory.c -index db64116a4f84..3e0266c973e1 100644 +index 3d3ebdc002d5..9634e7698e76 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -64,7 +64,11 @@ unsigned long transparent_hugepage_flags __read_mostly = @@ -13730,20 +12430,8 @@ index db64116a4f84..3e0266c973e1 100644 (1<> (20 - PAGE_SHIFT); /* Use a smaller cluster for small-memory machines */ -@@ -1060,4 +1064,5 @@ void __init swap_setup(void) +@@ -1092,4 +1096,5 @@ void __init swap_setup(void) * Right now other parts of the system means that we * _really_ don't want to cluster much more */ @@ -13824,38 +12512,10 @@ index bd5183dfd879..3a410f53a07c 100644 /* diff --git a/mm/vmscan.c b/mm/vmscan.c -index b1ec5ece067e..e258174d240a 100644 +index c767d71c43d7..d10fbd4c901f 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c -@@ -148,6 +148,15 @@ struct scan_control { - /* The file folios on the current node are dangerously low */ - unsigned int file_is_tiny:1; - -+ /* The anonymous pages on the current node are below vm.anon_min_ratio */ -+ unsigned int anon_below_min:1; -+ -+ /* The clean file pages on the current node are below vm.clean_low_ratio */ -+ unsigned int clean_below_low:1; -+ -+ /* The clean file pages on the current node are below vm.clean_min_ratio */ -+ unsigned int clean_below_min:1; -+ - /* Always discard instead of demoting to lower tier memory */ - unsigned int no_demotion:1; - -@@ -197,10 +206,23 @@ struct scan_control { - #define prefetchw_prev_lru_folio(_folio, _base, _field) do { } while (0) - #endif - -+bool sysctl_workingset_protection __read_mostly = true; -+u8 sysctl_anon_min_ratio __read_mostly = CONFIG_ANON_MIN_RATIO; -+u8 sysctl_clean_low_ratio __read_mostly = CONFIG_CLEAN_LOW_RATIO; -+u8 sysctl_clean_min_ratio __read_mostly = CONFIG_CLEAN_MIN_RATIO; -+static u64 sysctl_anon_min_ratio_kb __read_mostly = 0; -+static u64 sysctl_clean_low_ratio_kb __read_mostly = 0; -+static u64 sysctl_clean_min_ratio_kb __read_mostly = 0; -+static u64 workingset_protection_prev_totalram __read_mostly = 0; -+ +@@ -200,7 +200,11 @@ struct scan_control { /* * From 0 .. MAX_SWAPPINESS. Higher means more swappy. */ @@ -13867,178 +12527,8 @@ index b1ec5ece067e..e258174d240a 100644 #ifdef CONFIG_MEMCG -@@ -1097,6 +1119,10 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, - folio_mapped(folio) && folio_test_referenced(folio)) - goto keep_locked; - -+ if (folio_is_file_lru(folio) ? sc->clean_below_min : -+ (sc->anon_below_min && !sc->clean_below_min)) -+ goto keep_locked; -+ - /* - * The number of dirty pages determines if a node is marked - * reclaim_congested. kswapd will stall and start writing -@@ -2423,6 +2449,15 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, - goto out; - } - -+ /* -+ * Force-scan anon if clean file pages is under vm.clean_low_ratio -+ * or vm.clean_min_ratio. -+ */ -+ if (sc->clean_below_low || sc->clean_below_min) { -+ scan_balance = SCAN_ANON; -+ goto out; -+ } -+ - /* - * If there is enough inactive page cache, we do not reclaim - * anything from the anonymous working right now. -@@ -2567,6 +2602,14 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, - BUG(); - } - -+ /* -+ * Hard protection of the working set. -+ * Don't reclaim anon/file pages when the amount is -+ * below the watermark of the same type. -+ */ -+ if (file ? sc->clean_below_min : sc->anon_below_min) -+ scan = 0; -+ - nr[lru] = scan; - } - } -@@ -2586,6 +2629,96 @@ static bool can_age_anon_pages(struct pglist_data *pgdat, - return can_demote(pgdat->node_id, sc); - } - -+int vm_workingset_protection_update_handler(const struct ctl_table *table, int write, -+ void __user *buffer, size_t *lenp, loff_t *ppos) -+{ -+ int ret = proc_dou8vec_minmax(table, write, buffer, lenp, ppos); -+ if (ret || !write) -+ return ret; -+ -+ workingset_protection_prev_totalram = 0; -+ -+ return 0; -+} -+ -+static void prepare_workingset_protection(pg_data_t *pgdat, struct scan_control *sc) -+{ -+ unsigned long node_mem_total; -+ struct sysinfo i; -+ -+ if (!(sysctl_workingset_protection)) { -+ sc->anon_below_min = 0; -+ sc->clean_below_low = 0; -+ sc->clean_below_min = 0; -+ return; -+ } -+ -+ if (likely(sysctl_anon_min_ratio || -+ sysctl_clean_low_ratio || -+ sysctl_clean_min_ratio)) { -+#ifdef CONFIG_NUMA -+ si_meminfo_node(&i, pgdat->node_id); -+#else //CONFIG_NUMA -+ si_meminfo(&i); -+#endif //CONFIG_NUMA -+ node_mem_total = i.totalram; -+ -+ if (unlikely(workingset_protection_prev_totalram != node_mem_total)) { -+ sysctl_anon_min_ratio_kb = -+ node_mem_total * sysctl_anon_min_ratio / 100; -+ sysctl_clean_low_ratio_kb = -+ node_mem_total * sysctl_clean_low_ratio / 100; -+ sysctl_clean_min_ratio_kb = -+ node_mem_total * sysctl_clean_min_ratio / 100; -+ workingset_protection_prev_totalram = node_mem_total; -+ } -+ } -+ -+ /* -+ * Check the number of anonymous pages to protect them from -+ * reclaiming if their amount is below the specified. -+ */ -+ if (sysctl_anon_min_ratio) { -+ unsigned long reclaimable_anon; -+ -+ reclaimable_anon = -+ node_page_state(pgdat, NR_ACTIVE_ANON) + -+ node_page_state(pgdat, NR_INACTIVE_ANON) + -+ node_page_state(pgdat, NR_ISOLATED_ANON); -+ -+ sc->anon_below_min = reclaimable_anon < sysctl_anon_min_ratio_kb; -+ } else -+ sc->anon_below_min = 0; -+ -+ /* -+ * Check the number of clean file pages to protect them from -+ * reclaiming if their amount is below the specified. -+ */ -+ if (sysctl_clean_low_ratio || sysctl_clean_min_ratio) { -+ unsigned long reclaimable_file, dirty, clean; -+ -+ reclaimable_file = -+ node_page_state(pgdat, NR_ACTIVE_FILE) + -+ node_page_state(pgdat, NR_INACTIVE_FILE) + -+ node_page_state(pgdat, NR_ISOLATED_FILE); -+ dirty = node_page_state(pgdat, NR_FILE_DIRTY); -+ /* -+ * node_page_state() sum can go out of sync since -+ * all the values are not read at once. -+ */ -+ if (likely(reclaimable_file > dirty)) -+ clean = reclaimable_file - dirty; -+ else -+ clean = 0; -+ -+ sc->clean_below_low = clean < sysctl_clean_low_ratio_kb; -+ sc->clean_below_min = clean < sysctl_clean_min_ratio_kb; -+ } else { -+ sc->clean_below_low = 0; -+ sc->clean_below_min = 0; -+ } -+} -+ - #ifdef CONFIG_LRU_GEN - - #ifdef CONFIG_LRU_GEN_ENABLED -@@ -4539,6 +4672,12 @@ static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int sw - */ - if (!swappiness) - type = LRU_GEN_FILE; -+ else if (sc->clean_below_min) -+ type = LRU_GEN_ANON; -+ else if (sc->anon_below_min) -+ type = LRU_GEN_FILE; -+ else if (sc->clean_below_low) -+ type = LRU_GEN_ANON; - else if (min_seq[LRU_GEN_ANON] < min_seq[LRU_GEN_FILE]) - type = LRU_GEN_ANON; - else if (swappiness == 1) -@@ -4829,6 +4968,8 @@ static int shrink_one(struct lruvec *lruvec, struct scan_control *sc) - struct mem_cgroup *memcg = lruvec_memcg(lruvec); - struct pglist_data *pgdat = lruvec_pgdat(lruvec); - -+ prepare_workingset_protection(pgdat, sc); -+ - /* lru_gen_age_node() called mem_cgroup_calculate_protection() */ - if (mem_cgroup_below_min(NULL, memcg)) - return MEMCG_LRU_YOUNG; -@@ -5977,6 +6118,8 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) - - prepare_scan_control(pgdat, sc); - -+ prepare_workingset_protection(pgdat, sc); -+ - shrink_node_memcgs(pgdat, sc); - - flush_reclaim_state(sc); diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c -index 6872b5aff73e..1910fe1b2471 100644 +index e4decfb270fa..38bff2d8a740 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -634,7 +634,7 @@ static int inet_csk_wait_for_connect(struct sock *sk, long timeo) @@ -14051,805 +12541,25 @@ index 6872b5aff73e..1910fe1b2471 100644 release_sock(sk); if (reqsk_queue_empty(&icsk->icsk_accept_queue)) -- -2.48.0.rc1 +2.48.1 -From 6fe0c5e4544df7c64f9cdca4295e1c0e0c45740e Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Sun, 2 Feb 2025 13:54:07 +0100 -Subject: [PATCH 05/12] crypto +From e4e4fc74df918a945d88390ed011a47640bdc9c7 Mon Sep 17 00:00:00 2001 +From: Eric Naim +Date: Mon, 3 Feb 2025 11:05:11 +0800 +Subject: [PATCH 5/6] fixes -Signed-off-by: Peter Jung +Signed-off-by: Eric Naim --- - arch/x86/crypto/aes-gcm-avx10-x86_64.S | 119 ++++----- - arch/x86/crypto/aes-xts-avx-x86_64.S | 329 +++++++++++++------------ - arch/x86/crypto/aesni-intel_glue.c | 10 +- - 3 files changed, 221 insertions(+), 237 deletions(-) - -diff --git a/arch/x86/crypto/aes-gcm-avx10-x86_64.S b/arch/x86/crypto/aes-gcm-avx10-x86_64.S -index 97e0ee515fc5..02ee11083d4f 100644 ---- a/arch/x86/crypto/aes-gcm-avx10-x86_64.S -+++ b/arch/x86/crypto/aes-gcm-avx10-x86_64.S -@@ -88,7 +88,7 @@ - - // A shuffle mask that reflects the bytes of 16-byte blocks - .Lbswap_mask: -- .octa 0x000102030405060708090a0b0c0d0e0f -+ .octa 0x000102030405060708090a0b0c0d0e0f - - // This is the GHASH reducing polynomial without its constant term, i.e. - // x^128 + x^7 + x^2 + x, represented using the backwards mapping -@@ -384,8 +384,8 @@ - vpshufd $0xd3, H_CUR_XMM, %xmm0 - vpsrad $31, %xmm0, %xmm0 - vpaddq H_CUR_XMM, H_CUR_XMM, H_CUR_XMM -- vpand .Lgfpoly_and_internal_carrybit(%rip), %xmm0, %xmm0 -- vpxor %xmm0, H_CUR_XMM, H_CUR_XMM -+ // H_CUR_XMM ^= xmm0 & gfpoly_and_internal_carrybit -+ vpternlogd $0x78, .Lgfpoly_and_internal_carrybit(%rip), %xmm0, H_CUR_XMM - - // Load the gfpoly constant. - vbroadcasti32x4 .Lgfpoly(%rip), GFPOLY -@@ -562,6 +562,32 @@ - vpxord RNDKEY0, V3, V3 - .endm - -+// Do the last AES round for four vectors of counter blocks V0-V3, XOR source -+// data with the resulting keystream, and write the result to DST and -+// GHASHDATA[0-3]. (Implementation differs slightly, but has the same effect.) -+.macro _aesenclast_and_xor_4x -+ // XOR the source data with the last round key, saving the result in -+ // GHASHDATA[0-3]. This reduces latency by taking advantage of the -+ // property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a). -+ vpxord 0*VL(SRC), RNDKEYLAST, GHASHDATA0 -+ vpxord 1*VL(SRC), RNDKEYLAST, GHASHDATA1 -+ vpxord 2*VL(SRC), RNDKEYLAST, GHASHDATA2 -+ vpxord 3*VL(SRC), RNDKEYLAST, GHASHDATA3 -+ -+ // Do the last AES round. This handles the XOR with the source data -+ // too, as per the optimization described above. -+ vaesenclast GHASHDATA0, V0, GHASHDATA0 -+ vaesenclast GHASHDATA1, V1, GHASHDATA1 -+ vaesenclast GHASHDATA2, V2, GHASHDATA2 -+ vaesenclast GHASHDATA3, V3, GHASHDATA3 -+ -+ // Store the en/decrypted data to DST. -+ vmovdqu8 GHASHDATA0, 0*VL(DST) -+ vmovdqu8 GHASHDATA1, 1*VL(DST) -+ vmovdqu8 GHASHDATA2, 2*VL(DST) -+ vmovdqu8 GHASHDATA3, 3*VL(DST) -+.endm -+ - // void aes_gcm_{enc,dec}_update_##suffix(const struct aes_gcm_key_avx10 *key, - // const u32 le_ctr[4], u8 ghash_acc[16], - // const u8 *src, u8 *dst, int datalen); -@@ -640,7 +666,7 @@ - // LE_CTR contains the next set of little-endian counter blocks. - .set LE_CTR, V12 - -- // RNDKEY0, RNDKEYLAST, and RNDKEY_M[9-5] contain cached AES round keys, -+ // RNDKEY0, RNDKEYLAST, and RNDKEY_M[9-1] contain cached AES round keys, - // copied to all 128-bit lanes. RNDKEY0 is the zero-th round key, - // RNDKEYLAST the last, and RNDKEY_M\i the one \i-th from the last. - .set RNDKEY0, V13 -@@ -650,15 +676,10 @@ - .set RNDKEY_M7, V17 - .set RNDKEY_M6, V18 - .set RNDKEY_M5, V19 -- -- // RNDKEYLAST[0-3] temporarily store the last AES round key XOR'd with -- // the corresponding block of source data. This is useful because -- // vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a), and key ^ b can -- // be computed in parallel with the AES rounds. -- .set RNDKEYLAST0, V20 -- .set RNDKEYLAST1, V21 -- .set RNDKEYLAST2, V22 -- .set RNDKEYLAST3, V23 -+ .set RNDKEY_M4, V20 -+ .set RNDKEY_M3, V21 -+ .set RNDKEY_M2, V22 -+ .set RNDKEY_M1, V23 - - // GHASHTMP[0-2] are temporary variables used by _ghash_step_4x. These - // cannot coincide with anything used for AES encryption, since for -@@ -713,7 +734,7 @@ - // Pre-subtracting 4*VL from DATALEN saves an instruction from the main - // loop and also ensures that at least one write always occurs to - // DATALEN, zero-extending it and allowing DATALEN64 to be used later. -- sub $4*VL, DATALEN -+ add $-4*VL, DATALEN // shorter than 'sub 4*VL' when VL=32 - jl .Lcrypt_loop_4x_done\@ - - // Load powers of the hash key. -@@ -748,26 +769,15 @@ - add $16, %rax - cmp %rax, RNDKEYLAST_PTR - jne 1b -- vpxord 0*VL(SRC), RNDKEYLAST, RNDKEYLAST0 -- vpxord 1*VL(SRC), RNDKEYLAST, RNDKEYLAST1 -- vpxord 2*VL(SRC), RNDKEYLAST, RNDKEYLAST2 -- vpxord 3*VL(SRC), RNDKEYLAST, RNDKEYLAST3 -- vaesenclast RNDKEYLAST0, V0, GHASHDATA0 -- vaesenclast RNDKEYLAST1, V1, GHASHDATA1 -- vaesenclast RNDKEYLAST2, V2, GHASHDATA2 -- vaesenclast RNDKEYLAST3, V3, GHASHDATA3 -- vmovdqu8 GHASHDATA0, 0*VL(DST) -- vmovdqu8 GHASHDATA1, 1*VL(DST) -- vmovdqu8 GHASHDATA2, 2*VL(DST) -- vmovdqu8 GHASHDATA3, 3*VL(DST) -- add $4*VL, SRC -- add $4*VL, DST -- sub $4*VL, DATALEN -+ _aesenclast_and_xor_4x -+ sub $-4*VL, SRC // shorter than 'add 4*VL' when VL=32 -+ sub $-4*VL, DST -+ add $-4*VL, DATALEN - jl .Lghash_last_ciphertext_4x\@ - .endif - - // Cache as many additional AES round keys as possible. --.irp i, 9,8,7,6,5 -+.irp i, 9,8,7,6,5,4,3,2,1 - vbroadcasti32x4 -\i*16(RNDKEYLAST_PTR), RNDKEY_M\i - .endr - -@@ -799,50 +809,17 @@ - _vaesenc_4x RNDKEY - 128: - -- // XOR the source data with the last round key, saving the result in -- // RNDKEYLAST[0-3]. This reduces latency by taking advantage of the -- // property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a). --.if \enc -- vpxord 0*VL(SRC), RNDKEYLAST, RNDKEYLAST0 -- vpxord 1*VL(SRC), RNDKEYLAST, RNDKEYLAST1 -- vpxord 2*VL(SRC), RNDKEYLAST, RNDKEYLAST2 -- vpxord 3*VL(SRC), RNDKEYLAST, RNDKEYLAST3 --.else -- vpxord GHASHDATA0, RNDKEYLAST, RNDKEYLAST0 -- vpxord GHASHDATA1, RNDKEYLAST, RNDKEYLAST1 -- vpxord GHASHDATA2, RNDKEYLAST, RNDKEYLAST2 -- vpxord GHASHDATA3, RNDKEYLAST, RNDKEYLAST3 --.endif -- - // Finish the AES encryption of the counter blocks in V0-V3, interleaved - // with the GHASH update of the ciphertext blocks in GHASHDATA[0-3]. --.irp i, 9,8,7,6,5 -+.irp i, 9,8,7,6,5,4,3,2,1 -+ _ghash_step_4x (9 - \i) - _vaesenc_4x RNDKEY_M\i -- _ghash_step_4x (9 - \i) --.endr --.irp i, 4,3,2,1 -- vbroadcasti32x4 -\i*16(RNDKEYLAST_PTR), RNDKEY -- _vaesenc_4x RNDKEY -- _ghash_step_4x (9 - \i) - .endr - _ghash_step_4x 9 -- -- // Do the last AES round. This handles the XOR with the source data -- // too, as per the optimization described above. -- vaesenclast RNDKEYLAST0, V0, GHASHDATA0 -- vaesenclast RNDKEYLAST1, V1, GHASHDATA1 -- vaesenclast RNDKEYLAST2, V2, GHASHDATA2 -- vaesenclast RNDKEYLAST3, V3, GHASHDATA3 -- -- // Store the en/decrypted data to DST. -- vmovdqu8 GHASHDATA0, 0*VL(DST) -- vmovdqu8 GHASHDATA1, 1*VL(DST) -- vmovdqu8 GHASHDATA2, 2*VL(DST) -- vmovdqu8 GHASHDATA3, 3*VL(DST) -- -- add $4*VL, SRC -- add $4*VL, DST -- sub $4*VL, DATALEN -+ _aesenclast_and_xor_4x -+ sub $-4*VL, SRC // shorter than 'add 4*VL' when VL=32 -+ sub $-4*VL, DST -+ add $-4*VL, DATALEN - jge .Lcrypt_loop_4x\@ - - .if \enc -@@ -856,7 +833,7 @@ - .Lcrypt_loop_4x_done\@: - - // Undo the extra subtraction by 4*VL and check whether data remains. -- add $4*VL, DATALEN -+ sub $-4*VL, DATALEN // shorter than 'add 4*VL' when VL=32 - jz .Ldone\@ - - // The data length isn't a multiple of 4*VL. Process the remaining data -@@ -940,7 +917,7 @@ - // GHASH. However, any such blocks are all-zeroes, and the values that - // they're multiplied with are also all-zeroes. Therefore they just add - // 0 * 0 = 0 to the final GHASH result, which makes no difference. -- vmovdqu8 (POWERS_PTR), H_POW1 -+ vmovdqu8 (POWERS_PTR), H_POW1 - .if \enc - vmovdqu8 V0, V1{%k1}{z} - .endif -diff --git a/arch/x86/crypto/aes-xts-avx-x86_64.S b/arch/x86/crypto/aes-xts-avx-x86_64.S -index 48f97b79f7a9..8a3e23fbcf85 100644 ---- a/arch/x86/crypto/aes-xts-avx-x86_64.S -+++ b/arch/x86/crypto/aes-xts-avx-x86_64.S -@@ -80,22 +80,6 @@ - .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 - .text - --// Function parameters --.set KEY, %rdi // Initially points to crypto_aes_ctx, then is -- // advanced to point to 7th-from-last round key --.set SRC, %rsi // Pointer to next source data --.set DST, %rdx // Pointer to next destination data --.set LEN, %ecx // Remaining length in bytes --.set LEN8, %cl --.set LEN64, %rcx --.set TWEAK, %r8 // Pointer to next tweak -- --// %rax holds the AES key length in bytes. --.set KEYLEN, %eax --.set KEYLEN64, %rax -- --// %r9-r11 are available as temporaries. -- - .macro _define_Vi i - .if VL == 16 - .set V\i, %xmm\i -@@ -112,41 +96,31 @@ - // Define register aliases V0-V15, or V0-V31 if all 32 SIMD registers - // are available, that map to the xmm, ymm, or zmm registers according - // to the selected Vector Length (VL). -- _define_Vi 0 -- _define_Vi 1 -- _define_Vi 2 -- _define_Vi 3 -- _define_Vi 4 -- _define_Vi 5 -- _define_Vi 6 -- _define_Vi 7 -- _define_Vi 8 -- _define_Vi 9 -- _define_Vi 10 -- _define_Vi 11 -- _define_Vi 12 -- _define_Vi 13 -- _define_Vi 14 -- _define_Vi 15 -+.irp i, 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 -+ _define_Vi \i -+.endr - .if USE_AVX10 -- _define_Vi 16 -- _define_Vi 17 -- _define_Vi 18 -- _define_Vi 19 -- _define_Vi 20 -- _define_Vi 21 -- _define_Vi 22 -- _define_Vi 23 -- _define_Vi 24 -- _define_Vi 25 -- _define_Vi 26 -- _define_Vi 27 -- _define_Vi 28 -- _define_Vi 29 -- _define_Vi 30 -- _define_Vi 31 -+.irp i, 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 -+ _define_Vi \i -+.endr - .endif - -+ // Function parameters -+ .set KEY, %rdi // Initially points to crypto_aes_ctx, then is -+ // advanced to point to 7th-from-last round key -+ .set SRC, %rsi // Pointer to next source data -+ .set DST, %rdx // Pointer to next destination data -+ .set LEN, %ecx // Remaining length in bytes -+ .set LEN8, %cl -+ .set LEN64, %rcx -+ .set TWEAK, %r8 // Pointer to next tweak -+ -+ // %rax holds the AES key length in bytes. -+ .set KEYLEN, %eax -+ .set KEYLEN64, %rax -+ -+ // %r9-r11 are available as temporaries. -+ - // V0-V3 hold the data blocks during the main loop, or temporary values - // otherwise. V4-V5 hold temporary values. - -@@ -214,6 +188,7 @@ - .endm - - // Move a vector between memory and a register. -+// The register operand must be in the first 16 vector registers. - .macro _vmovdqu src, dst - .if VL < 64 - vmovdqu \src, \dst -@@ -234,11 +209,12 @@ - .endm - - // XOR two vectors together. -+// Any register operands must be in the first 16 vector registers. - .macro _vpxor src1, src2, dst --.if USE_AVX10 -- vpxord \src1, \src2, \dst --.else -+.if VL < 64 - vpxor \src1, \src2, \dst -+.else -+ vpxord \src1, \src2, \dst - .endif - .endm - -@@ -259,8 +235,12 @@ - vpshufd $0x13, \src, \tmp - vpaddq \src, \src, \dst - vpsrad $31, \tmp, \tmp -+.if USE_AVX10 -+ vpternlogd $0x78, GF_POLY_XMM, \tmp, \dst -+.else - vpand GF_POLY_XMM, \tmp, \tmp - vpxor \tmp, \dst, \dst -+.endif - .endm - - // Given the XTS tweak(s) in the vector \src, compute the next vector of -@@ -369,9 +349,14 @@ - - // Do one step in computing the next set of tweaks using the VPCLMULQDQ method - // (the same method _next_tweakvec uses for VL > 16). This means multiplying --// each tweak by x^(4*VL/16) independently. Since 4*VL/16 is a multiple of 8 --// when VL > 16 (which it is here), the needed shift amounts are byte-aligned, --// which allows the use of vpsrldq and vpslldq to do 128-bit wide shifts. -+// each tweak by x^(4*VL/16) independently. -+// -+// Since 4*VL/16 is a multiple of 8 when VL > 16 (which it is here), the needed -+// shift amounts are byte-aligned, which allows the use of vpsrldq and vpslldq -+// to do 128-bit wide shifts. The 128-bit left shift (vpslldq) saves -+// instructions directly. The 128-bit right shift (vpsrldq) performs better -+// than a 64-bit right shift on Intel CPUs in the context where it is used here, -+// because it runs on a different execution port from the AES instructions. - .macro _tweak_step_pclmul i - .if \i == 0 - vpsrldq $(128 - 4*VL/16) / 8, TWEAK0, NEXT_TWEAK0 -@@ -406,7 +391,7 @@ - // \i that include at least 0 through 19, then 1000 which signals the last step. - // - // This is used to interleave the computation of the next set of tweaks with the --// AES en/decryptions, which increases performance in some cases. -+// AES en/decryptions, which increases performance in some cases. Clobbers V5. - .macro _tweak_step i - .if VL == 16 - _tweak_step_mulx \i -@@ -443,9 +428,10 @@ - // the last round needs different instructions. - // - // An alternative approach would be to roll up all the round loops. We -- // don't do that because it isn't compatible with caching the round keys -- // in registers which we do when possible (see below), and also because -- // it seems unwise to rely *too* heavily on the CPU's branch predictor. -+ // don't do that because (a) it isn't compatible with caching the round -+ // keys in registers which we do when possible (see below), (b) we -+ // interleave the AES rounds with the XTS tweak computation, and (c) it -+ // seems unwise to rely *too* heavily on the CPU's branch predictor. - lea OFFS-16(KEY, KEYLEN64, 4), KEY - - // If all 32 SIMD registers are available, cache all the round keys. -@@ -472,90 +458,94 @@ - .endif - .endm - --// Do a single round of AES encryption (if \enc==1) or decryption (if \enc==0) --// on the block(s) in \data using the round key(s) in \key. The register length --// determines the number of AES blocks en/decrypted. --.macro _vaes enc, last, key, data -+// Do a single non-last round of AES encryption (if \enc==1) or decryption (if -+// \enc==0) on the block(s) in \data using the round key(s) in \key. The -+// register length determines the number of AES blocks en/decrypted. -+.macro _vaes enc, key, data - .if \enc --.if \last -- vaesenclast \key, \data, \data --.else - vaesenc \key, \data, \data --.endif --.else --.if \last -- vaesdeclast \key, \data, \data - .else - vaesdec \key, \data, \data - .endif -+.endm -+ -+// Same as _vaes, but does the last round. -+.macro _vaeslast enc, key, data -+.if \enc -+ vaesenclast \key, \data, \data -+.else -+ vaesdeclast \key, \data, \data - .endif - .endm - --// Do a single round of AES en/decryption on the block(s) in \data, using the --// same key for all block(s). The round key is loaded from the appropriate --// register or memory location for round \i. May clobber V4. --.macro _vaes_1x enc, last, i, xmm_suffix, data -+// Do a single non-last round of AES en/decryption on the block(s) in \data, -+// using the same key for all block(s). The round key is loaded from the -+// appropriate register or memory location for round \i. May clobber \tmp. -+.macro _vaes_1x enc, i, xmm_suffix, data, tmp - .if USE_AVX10 -- _vaes \enc, \last, KEY\i\xmm_suffix, \data -+ _vaes \enc, KEY\i\xmm_suffix, \data - .else - .ifnb \xmm_suffix -- _vaes \enc, \last, (\i-7)*16(KEY), \data -+ _vaes \enc, (\i-7)*16(KEY), \data - .else -- _vbroadcast128 (\i-7)*16(KEY), V4 -- _vaes \enc, \last, V4, \data -+ _vbroadcast128 (\i-7)*16(KEY), \tmp -+ _vaes \enc, \tmp, \data - .endif - .endif - .endm - --// Do a single round of AES en/decryption on the blocks in registers V0-V3, --// using the same key for all blocks. The round key is loaded from the -+// Do a single non-last round of AES en/decryption on the blocks in registers -+// V0-V3, using the same key for all blocks. The round key is loaded from the - // appropriate register or memory location for round \i. In addition, does two --// steps of the computation of the next set of tweaks. May clobber V4. --.macro _vaes_4x enc, last, i -+// steps of the computation of the next set of tweaks. May clobber V4 and V5. -+.macro _vaes_4x enc, i - .if USE_AVX10 - _tweak_step (2*(\i-5)) -- _vaes \enc, \last, KEY\i, V0 -- _vaes \enc, \last, KEY\i, V1 -+ _vaes \enc, KEY\i, V0 -+ _vaes \enc, KEY\i, V1 - _tweak_step (2*(\i-5) + 1) -- _vaes \enc, \last, KEY\i, V2 -- _vaes \enc, \last, KEY\i, V3 -+ _vaes \enc, KEY\i, V2 -+ _vaes \enc, KEY\i, V3 - .else - _vbroadcast128 (\i-7)*16(KEY), V4 - _tweak_step (2*(\i-5)) -- _vaes \enc, \last, V4, V0 -- _vaes \enc, \last, V4, V1 -+ _vaes \enc, V4, V0 -+ _vaes \enc, V4, V1 - _tweak_step (2*(\i-5) + 1) -- _vaes \enc, \last, V4, V2 -- _vaes \enc, \last, V4, V3 -+ _vaes \enc, V4, V2 -+ _vaes \enc, V4, V3 - .endif - .endm - - // Do tweaked AES en/decryption (i.e., XOR with \tweak, then AES en/decrypt, - // then XOR with \tweak again) of the block(s) in \data. To process a single - // block, use xmm registers and set \xmm_suffix=_XMM. To process a vector of --// length VL, use V* registers and leave \xmm_suffix empty. May clobber V4. --.macro _aes_crypt enc, xmm_suffix, tweak, data -+// length VL, use V* registers and leave \xmm_suffix empty. Clobbers \tmp. -+.macro _aes_crypt enc, xmm_suffix, tweak, data, tmp - _xor3 KEY0\xmm_suffix, \tweak, \data - cmp $24, KEYLEN - jl .Laes128\@ - je .Laes192\@ -- _vaes_1x \enc, 0, 1, \xmm_suffix, \data -- _vaes_1x \enc, 0, 2, \xmm_suffix, \data -+ _vaes_1x \enc, 1, \xmm_suffix, \data, tmp=\tmp -+ _vaes_1x \enc, 2, \xmm_suffix, \data, tmp=\tmp - .Laes192\@: -- _vaes_1x \enc, 0, 3, \xmm_suffix, \data -- _vaes_1x \enc, 0, 4, \xmm_suffix, \data -+ _vaes_1x \enc, 3, \xmm_suffix, \data, tmp=\tmp -+ _vaes_1x \enc, 4, \xmm_suffix, \data, tmp=\tmp - .Laes128\@: -- _vaes_1x \enc, 0, 5, \xmm_suffix, \data -- _vaes_1x \enc, 0, 6, \xmm_suffix, \data -- _vaes_1x \enc, 0, 7, \xmm_suffix, \data -- _vaes_1x \enc, 0, 8, \xmm_suffix, \data -- _vaes_1x \enc, 0, 9, \xmm_suffix, \data -- _vaes_1x \enc, 0, 10, \xmm_suffix, \data -- _vaes_1x \enc, 0, 11, \xmm_suffix, \data -- _vaes_1x \enc, 0, 12, \xmm_suffix, \data -- _vaes_1x \enc, 0, 13, \xmm_suffix, \data -- _vaes_1x \enc, 1, 14, \xmm_suffix, \data -- _vpxor \tweak, \data, \data -+.irp i, 5,6,7,8,9,10,11,12,13 -+ _vaes_1x \enc, \i, \xmm_suffix, \data, tmp=\tmp -+.endr -+.if USE_AVX10 -+ vpxord KEY14\xmm_suffix, \tweak, \tmp -+.else -+.ifnb \xmm_suffix -+ vpxor 7*16(KEY), \tweak, \tmp -+.else -+ _vbroadcast128 7*16(KEY), \tmp -+ vpxor \tweak, \tmp, \tmp -+.endif -+.endif -+ _vaeslast \enc, \tmp, \data - .endm - - .macro _aes_xts_crypt enc -@@ -581,7 +571,7 @@ - // Compute the first set of tweaks TWEAK[0-3]. - _compute_first_set_of_tweaks - -- sub $4*VL, LEN -+ add $-4*VL, LEN // shorter than 'sub 4*VL' when VL=32 - jl .Lhandle_remainder\@ - - .Lmain_loop\@: -@@ -589,10 +579,10 @@ - - // XOR each source block with its tweak and the zero-th round key. - .if USE_AVX10 -- vmovdqu8 0*VL(SRC), V0 -- vmovdqu8 1*VL(SRC), V1 -- vmovdqu8 2*VL(SRC), V2 -- vmovdqu8 3*VL(SRC), V3 -+ _vmovdqu 0*VL(SRC), V0 -+ _vmovdqu 1*VL(SRC), V1 -+ _vmovdqu 2*VL(SRC), V2 -+ _vmovdqu 3*VL(SRC), V3 - vpternlogd $0x96, TWEAK0, KEY0, V0 - vpternlogd $0x96, TWEAK1, KEY0, V1 - vpternlogd $0x96, TWEAK2, KEY0, V2 -@@ -612,28 +602,43 @@ - je .Laes192\@ - // Do all the AES rounds on the data blocks, interleaved with - // the computation of the next set of tweaks. -- _vaes_4x \enc, 0, 1 -- _vaes_4x \enc, 0, 2 -+ _vaes_4x \enc, 1 -+ _vaes_4x \enc, 2 - .Laes192\@: -- _vaes_4x \enc, 0, 3 -- _vaes_4x \enc, 0, 4 -+ _vaes_4x \enc, 3 -+ _vaes_4x \enc, 4 - .Laes128\@: -- _vaes_4x \enc, 0, 5 -- _vaes_4x \enc, 0, 6 -- _vaes_4x \enc, 0, 7 -- _vaes_4x \enc, 0, 8 -- _vaes_4x \enc, 0, 9 -- _vaes_4x \enc, 0, 10 -- _vaes_4x \enc, 0, 11 -- _vaes_4x \enc, 0, 12 -- _vaes_4x \enc, 0, 13 -- _vaes_4x \enc, 1, 14 -- -- // XOR in the tweaks again. -- _vpxor TWEAK0, V0, V0 -- _vpxor TWEAK1, V1, V1 -- _vpxor TWEAK2, V2, V2 -- _vpxor TWEAK3, V3, V3 -+.irp i, 5,6,7,8,9,10,11,12,13 -+ _vaes_4x \enc, \i -+.endr -+ // Do the last AES round, then XOR the results with the tweaks again. -+ // Reduce latency by doing the XOR before the vaesenclast, utilizing the -+ // property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a) -+ // (and likewise for vaesdeclast). -+.if USE_AVX10 -+ _tweak_step 18 -+ _tweak_step 19 -+ vpxord TWEAK0, KEY14, V4 -+ vpxord TWEAK1, KEY14, V5 -+ _vaeslast \enc, V4, V0 -+ _vaeslast \enc, V5, V1 -+ vpxord TWEAK2, KEY14, V4 -+ vpxord TWEAK3, KEY14, V5 -+ _vaeslast \enc, V4, V2 -+ _vaeslast \enc, V5, V3 -+.else -+ _vbroadcast128 7*16(KEY), V4 -+ _tweak_step 18 // uses V5 -+ _tweak_step 19 // uses V5 -+ vpxor TWEAK0, V4, V5 -+ _vaeslast \enc, V5, V0 -+ vpxor TWEAK1, V4, V5 -+ _vaeslast \enc, V5, V1 -+ vpxor TWEAK2, V4, V5 -+ vpxor TWEAK3, V4, V4 -+ _vaeslast \enc, V5, V2 -+ _vaeslast \enc, V4, V3 -+.endif - - // Store the destination blocks. - _vmovdqu V0, 0*VL(DST) -@@ -644,9 +649,9 @@ - // Finish computing the next set of tweaks. - _tweak_step 1000 - -- add $4*VL, SRC -- add $4*VL, DST -- sub $4*VL, LEN -+ sub $-4*VL, SRC // shorter than 'add 4*VL' when VL=32 -+ sub $-4*VL, DST -+ add $-4*VL, LEN - jge .Lmain_loop\@ - - // Check for the uncommon case where the data length isn't a multiple of -@@ -670,7 +675,7 @@ - jl .Lvec_at_a_time_done\@ - .Lvec_at_a_time\@: - _vmovdqu (SRC), V0 -- _aes_crypt \enc, , TWEAK0, V0 -+ _aes_crypt \enc, , TWEAK0, V0, tmp=V1 - _vmovdqu V0, (DST) - _next_tweakvec TWEAK0, V0, V1, TWEAK0 - add $VL, SRC -@@ -687,7 +692,7 @@ - jl .Lblock_at_a_time_done\@ - .Lblock_at_a_time\@: - vmovdqu (SRC), %xmm0 -- _aes_crypt \enc, _XMM, TWEAK0_XMM, %xmm0 -+ _aes_crypt \enc, _XMM, TWEAK0_XMM, %xmm0, tmp=%xmm1 - vmovdqu %xmm0, (DST) - _next_tweak TWEAK0_XMM, %xmm0, TWEAK0_XMM - add $16, SRC -@@ -715,7 +720,7 @@ - // Do it now by advancing the tweak and decrypting the last full block. - _next_tweak TWEAK0_XMM, %xmm0, TWEAK1_XMM - vmovdqu (SRC), %xmm0 -- _aes_crypt \enc, _XMM, TWEAK1_XMM, %xmm0 -+ _aes_crypt \enc, _XMM, TWEAK1_XMM, %xmm0, tmp=%xmm1 - .endif - - .if USE_AVX10 -@@ -758,47 +763,49 @@ - vpblendvb %xmm3, %xmm0, %xmm1, %xmm0 - .endif - // En/decrypt again and store the last full block. -- _aes_crypt \enc, _XMM, TWEAK0_XMM, %xmm0 -+ _aes_crypt \enc, _XMM, TWEAK0_XMM, %xmm0, tmp=%xmm1 - vmovdqu %xmm0, (DST) - jmp .Ldone\@ - .endm - - // void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key, - // u8 iv[AES_BLOCK_SIZE]); -+// -+// Encrypt |iv| using the AES key |tweak_key| to get the first tweak. Assumes -+// that the CPU supports AES-NI and AVX, but not necessarily VAES or AVX10. - SYM_TYPED_FUNC_START(aes_xts_encrypt_iv) -- vmovdqu (%rsi), %xmm0 -- vpxor (%rdi), %xmm0, %xmm0 -- movl 480(%rdi), %eax // AES key length -- lea -16(%rdi, %rax, 4), %rdi -- cmp $24, %eax -+ .set TWEAK_KEY, %rdi -+ .set IV, %rsi -+ .set KEYLEN, %eax -+ .set KEYLEN64, %rax -+ -+ vmovdqu (IV), %xmm0 -+ vpxor (TWEAK_KEY), %xmm0, %xmm0 -+ movl 480(TWEAK_KEY), KEYLEN -+ lea -16(TWEAK_KEY, KEYLEN64, 4), TWEAK_KEY -+ cmp $24, KEYLEN - jl .Lencrypt_iv_aes128 - je .Lencrypt_iv_aes192 -- vaesenc -6*16(%rdi), %xmm0, %xmm0 -- vaesenc -5*16(%rdi), %xmm0, %xmm0 -+ vaesenc -6*16(TWEAK_KEY), %xmm0, %xmm0 -+ vaesenc -5*16(TWEAK_KEY), %xmm0, %xmm0 - .Lencrypt_iv_aes192: -- vaesenc -4*16(%rdi), %xmm0, %xmm0 -- vaesenc -3*16(%rdi), %xmm0, %xmm0 -+ vaesenc -4*16(TWEAK_KEY), %xmm0, %xmm0 -+ vaesenc -3*16(TWEAK_KEY), %xmm0, %xmm0 - .Lencrypt_iv_aes128: -- vaesenc -2*16(%rdi), %xmm0, %xmm0 -- vaesenc -1*16(%rdi), %xmm0, %xmm0 -- vaesenc 0*16(%rdi), %xmm0, %xmm0 -- vaesenc 1*16(%rdi), %xmm0, %xmm0 -- vaesenc 2*16(%rdi), %xmm0, %xmm0 -- vaesenc 3*16(%rdi), %xmm0, %xmm0 -- vaesenc 4*16(%rdi), %xmm0, %xmm0 -- vaesenc 5*16(%rdi), %xmm0, %xmm0 -- vaesenc 6*16(%rdi), %xmm0, %xmm0 -- vaesenclast 7*16(%rdi), %xmm0, %xmm0 -- vmovdqu %xmm0, (%rsi) -+.irp i, -2,-1,0,1,2,3,4,5,6 -+ vaesenc \i*16(TWEAK_KEY), %xmm0, %xmm0 -+.endr -+ vaesenclast 7*16(TWEAK_KEY), %xmm0, %xmm0 -+ vmovdqu %xmm0, (IV) - RET - SYM_FUNC_END(aes_xts_encrypt_iv) - - // Below are the actual AES-XTS encryption and decryption functions, - // instantiated from the above macro. They all have the following prototype: - // --// void (*xts_asm_func)(const struct crypto_aes_ctx *key, --// const u8 *src, u8 *dst, unsigned int len, --// u8 tweak[AES_BLOCK_SIZE]); -+// void (*xts_crypt_func)(const struct crypto_aes_ctx *key, -+// const u8 *src, u8 *dst, int len, -+// u8 tweak[AES_BLOCK_SIZE]); - // - // |key| is the data key. |tweak| contains the next tweak; the encryption of - // the original IV with the tweak key was already done. This function supports -diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c -index fbf43482e1f5..11e95fc62636 100644 ---- a/arch/x86/crypto/aesni-intel_glue.c -+++ b/arch/x86/crypto/aesni-intel_glue.c -@@ -505,7 +505,7 @@ static int xts_setkey_aesni(struct crypto_skcipher *tfm, const u8 *key, - typedef void (*xts_encrypt_iv_func)(const struct crypto_aes_ctx *tweak_key, - u8 iv[AES_BLOCK_SIZE]); - typedef void (*xts_crypt_func)(const struct crypto_aes_ctx *key, -- const u8 *src, u8 *dst, unsigned int len, -+ const u8 *src, u8 *dst, int len, - u8 tweak[AES_BLOCK_SIZE]); - - /* This handles cases where the source and/or destination span pages. */ -@@ -624,14 +624,14 @@ static void aesni_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key, - } - - static void aesni_xts_encrypt(const struct crypto_aes_ctx *key, -- const u8 *src, u8 *dst, unsigned int len, -+ const u8 *src, u8 *dst, int len, - u8 tweak[AES_BLOCK_SIZE]) - { - aesni_xts_enc(key, dst, src, len, tweak); - } - - static void aesni_xts_decrypt(const struct crypto_aes_ctx *key, -- const u8 *src, u8 *dst, unsigned int len, -+ const u8 *src, u8 *dst, int len, - u8 tweak[AES_BLOCK_SIZE]) - { - aesni_xts_dec(key, dst, src, len, tweak); -@@ -790,10 +790,10 @@ asmlinkage void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key, - \ - asmlinkage void \ - aes_xts_encrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src, \ -- u8 *dst, unsigned int len, u8 tweak[AES_BLOCK_SIZE]); \ -+ u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]); \ - asmlinkage void \ - aes_xts_decrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src, \ -- u8 *dst, unsigned int len, u8 tweak[AES_BLOCK_SIZE]); \ -+ u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]); \ - \ - static int xts_encrypt_##suffix(struct skcipher_request *req) \ - { \ --- -2.48.0.rc1 - -From 0be477e02bd37573754ab4fb4420929383b24b8c Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Sun, 2 Feb 2025 13:54:18 +0100 -Subject: [PATCH 06/12] fixes - -Signed-off-by: Peter Jung ---- - arch/Kconfig | 4 +- - arch/x86/boot/compressed/Makefile | 1 + - drivers/firmware/efi/libstub/Makefile | 2 +- - .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c | 12 +++-- - drivers/gpu/drm/drm_edid.c | 47 +++++++++++++++++-- - drivers/hid/hid-asus.c | 26 ++++++++++ - drivers/hid/hid-ids.h | 1 + - include/linux/platform_data/x86/asus-wmi.h | 5 ++ - kernel/fork.c | 9 ++-- - kernel/kprobes.c | 23 +++++---- - kernel/sched/ext.c | 7 +-- - scripts/package/PKGBUILD | 5 ++ - 12 files changed, 113 insertions(+), 29 deletions(-) + arch/Kconfig | 4 +-- + arch/x86/boot/compressed/Makefile | 1 + + drivers/gpu/drm/drm_edid.c | 47 +++++++++++++++++++++++++++++-- + drivers/hid/hid-ids.h | 1 + + kernel/sched/ext.c | 7 +++-- + scripts/package/PKGBUILD | 5 ++++ + 6 files changed, 57 insertions(+), 8 deletions(-) diff --git a/arch/Kconfig b/arch/Kconfig -index 6682b2a53e34..fe54298ae05c 100644 +index b8a4ff365582..9b087f9bb413 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -1137,7 +1137,7 @@ config ARCH_MMAP_RND_BITS @@ -14882,44 +12592,8 @@ index f2051644de94..606c74f27459 100644 KBUILD_CFLAGS += -fno-strict-aliasing -fPIE KBUILD_CFLAGS += -Wundef KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING -diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile -index ed4e8ddbe76a..1141cd06011f 100644 ---- a/drivers/firmware/efi/libstub/Makefile -+++ b/drivers/firmware/efi/libstub/Makefile -@@ -11,7 +11,7 @@ cflags-y := $(KBUILD_CFLAGS) - - cflags-$(CONFIG_X86_32) := -march=i386 - cflags-$(CONFIG_X86_64) := -mcmodel=small --cflags-$(CONFIG_X86) += -m$(BITS) -D__KERNEL__ \ -+cflags-$(CONFIG_X86) += -m$(BITS) -D__KERNEL__ -std=gnu11 \ - -fPIC -fno-strict-aliasing -mno-red-zone \ - -mno-mmx -mno-sse -fshort-wchar \ - -Wno-pointer-sign \ -diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c -index 382af92c4ff1..f836ab663568 100644 ---- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c -+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c -@@ -12272,10 +12272,14 @@ void amdgpu_dm_update_freesync_caps(struct drm_connector *connector, - - if (edid && (sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT || - sink->sink_signal == SIGNAL_TYPE_EDP)) { -- amdgpu_dm_connector->min_vfreq = connector->display_info.monitor_range.min_vfreq; -- amdgpu_dm_connector->max_vfreq = connector->display_info.monitor_range.max_vfreq; -- if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10) -- freesync_capable = true; -+ if (amdgpu_dm_connector->dc_link && -+ amdgpu_dm_connector->dc_link->dpcd_caps.allow_invalid_MSA_timing_param) { -+ amdgpu_dm_connector->min_vfreq = connector->display_info.monitor_range.min_vfreq; -+ amdgpu_dm_connector->max_vfreq = connector->display_info.monitor_range.max_vfreq; -+ if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10) -+ freesync_capable = true; -+ } -+ - parse_amd_vsdb(amdgpu_dm_connector, edid, &vsdb_info); - - if (vsdb_info.replay_mode) { diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c -index 855beafb76ff..ad78059ee954 100644 +index 13bc4c290b17..9b741e6262bc 100644 --- a/drivers/gpu/drm/drm_edid.c +++ b/drivers/gpu/drm/drm_edid.c @@ -94,6 +94,8 @@ static int oui(u8 first, u8 second, u8 third) @@ -14944,7 +12618,7 @@ index 855beafb76ff..ad78059ee954 100644 /* Sony PVM-2541A does up to 12 bpc, but only reports max 8 bpc */ EDID_QUIRK('S', 'N', 'Y', 0x2541, EDID_QUIRK_FORCE_12BPC), -@@ -6753,7 +6761,37 @@ static void update_display_info(struct drm_connector *connector, +@@ -6759,7 +6767,37 @@ static void update_display_info(struct drm_connector *connector, drm_edid_to_eld(connector, drm_edid); } @@ -14983,7 +12657,7 @@ index 855beafb76ff..ad78059ee954 100644 struct displayid_detailed_timings_1 *timings, bool type_7) { -@@ -6772,7 +6810,7 @@ static struct drm_display_mode *drm_mode_displayid_detailed(struct drm_device *d +@@ -6778,7 +6816,7 @@ static struct drm_display_mode *drm_mode_displayid_detailed(struct drm_device *d bool hsync_positive = (timings->hsync[1] >> 7) & 0x1; bool vsync_positive = (timings->vsync[1] >> 7) & 0x1; @@ -14992,7 +12666,7 @@ index 855beafb76ff..ad78059ee954 100644 if (!mode) return NULL; -@@ -6795,6 +6833,9 @@ static struct drm_display_mode *drm_mode_displayid_detailed(struct drm_device *d +@@ -6801,6 +6839,9 @@ static struct drm_display_mode *drm_mode_displayid_detailed(struct drm_device *d if (timings->flags & 0x80) mode->type |= DRM_MODE_TYPE_PREFERRED; @@ -15002,7 +12676,7 @@ index 855beafb76ff..ad78059ee954 100644 drm_mode_set_name(mode); return mode; -@@ -6817,7 +6858,7 @@ static int add_displayid_detailed_1_modes(struct drm_connector *connector, +@@ -6823,7 +6864,7 @@ static int add_displayid_detailed_1_modes(struct drm_connector *connector, for (i = 0; i < num_timings; i++) { struct displayid_detailed_timings_1 *timings = &det->timings[i]; @@ -15011,52 +12685,8 @@ index 855beafb76ff..ad78059ee954 100644 if (!newmode) continue; -diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c -index 506c6f377e7d..46e3e42f9eb5 100644 ---- a/drivers/hid/hid-asus.c -+++ b/drivers/hid/hid-asus.c -@@ -432,6 +432,26 @@ static int asus_kbd_get_functions(struct hid_device *hdev, - return ret; - } - -+static int asus_kbd_disable_oobe(struct hid_device *hdev) -+{ -+ const u8 init[][6] = { -+ { FEATURE_KBD_REPORT_ID, 0x05, 0x20, 0x31, 0x00, 0x08 }, -+ { FEATURE_KBD_REPORT_ID, 0xBA, 0xC5, 0xC4 }, -+ { FEATURE_KBD_REPORT_ID, 0xD0, 0x8F, 0x01 }, -+ { FEATURE_KBD_REPORT_ID, 0xD0, 0x85, 0xFF } -+ }; -+ int ret; -+ -+ for (size_t i = 0; i < ARRAY_SIZE(init); i++) { -+ ret = asus_kbd_set_report(hdev, init[i], sizeof(init[i])); -+ if (ret < 0) -+ return ret; -+ } -+ -+ hid_info(hdev, "Disabled OOBE for keyboard\n"); -+ return 0; -+} -+ - static void asus_schedule_work(struct asus_kbd_leds *led) - { - unsigned long flags; -@@ -534,6 +554,12 @@ static int asus_kbd_register_leds(struct hid_device *hdev) - ret = asus_kbd_init(hdev, FEATURE_KBD_LED_REPORT_ID2); - if (ret < 0) - return ret; -+ -+ if (dmi_match(DMI_PRODUCT_FAMILY, "ProArt P16")) { -+ ret = asus_kbd_disable_oobe(hdev); -+ if (ret < 0) -+ return ret; -+ } - } else { - /* Initialize keyboard */ - ret = asus_kbd_init(hdev, FEATURE_KBD_REPORT_ID); diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h -index d1d479ca50a2..d1ab021e4a6a 100644 +index c448de53bf91..6ee1790f2212 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -210,6 +210,7 @@ @@ -15067,123 +12697,17 @@ index d1d479ca50a2..d1ab021e4a6a 100644 #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY 0x1abe #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY_X 0x1b4c #define USB_DEVICE_ID_ASUSTEK_ROG_CLAYMORE_II_KEYBOARD 0x196b -diff --git a/include/linux/platform_data/x86/asus-wmi.h b/include/linux/platform_data/x86/asus-wmi.h -index 365e119bebaa..783e2a336861 100644 ---- a/include/linux/platform_data/x86/asus-wmi.h -+++ b/include/linux/platform_data/x86/asus-wmi.h -@@ -184,6 +184,11 @@ static const struct dmi_system_id asus_use_hid_led_dmi_ids[] = { - DMI_MATCH(DMI_PRODUCT_FAMILY, "ROG Flow"), - }, - }, -+ { -+ .matches = { -+ DMI_MATCH(DMI_PRODUCT_FAMILY, "ProArt P16"), -+ }, -+ }, - { - .matches = { - DMI_MATCH(DMI_BOARD_NAME, "GA403U"), -diff --git a/kernel/fork.c b/kernel/fork.c -index 0cb5431b4d7e..e919c8c3a121 100644 ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -1518,12 +1518,13 @@ struct file *get_task_exe_file(struct task_struct *task) - struct file *exe_file = NULL; - struct mm_struct *mm; - -+ if (task->flags & PF_KTHREAD) -+ return NULL; -+ - task_lock(task); - mm = task->mm; -- if (mm) { -- if (!(task->flags & PF_KTHREAD)) -- exe_file = get_mm_exe_file(mm); -- } -+ if (mm) -+ exe_file = get_mm_exe_file(mm); - task_unlock(task); - return exe_file; - } -diff --git a/kernel/kprobes.c b/kernel/kprobes.c -index b027a4030976..5cc750200f19 100644 ---- a/kernel/kprobes.c -+++ b/kernel/kprobes.c -@@ -1566,16 +1566,25 @@ static int check_kprobe_address_safe(struct kprobe *p, - if (ret) - return ret; - jump_label_lock(); -- preempt_disable(); - - /* Ensure the address is in a text area, and find a module if exists. */ - *probed_mod = NULL; - if (!core_kernel_text((unsigned long) p->addr)) { -+ guard(preempt)(); - *probed_mod = __module_text_address((unsigned long) p->addr); - if (!(*probed_mod)) { - ret = -EINVAL; - goto out; - } -+ -+ /* -+ * We must hold a refcount of the probed module while updating -+ * its code to prohibit unexpected unloading. -+ */ -+ if (unlikely(!try_module_get(*probed_mod))) { -+ ret = -ENOENT; -+ goto out; -+ } - } - /* Ensure it is not in reserved area. */ - if (in_gate_area_no_mm((unsigned long) p->addr) || -@@ -1584,21 +1593,13 @@ static int check_kprobe_address_safe(struct kprobe *p, - static_call_text_reserved(p->addr, p->addr) || - find_bug((unsigned long)p->addr) || - is_cfi_preamble_symbol((unsigned long)p->addr)) { -+ module_put(*probed_mod); - ret = -EINVAL; - goto out; - } - - /* Get module refcount and reject __init functions for loaded modules. */ - if (IS_ENABLED(CONFIG_MODULES) && *probed_mod) { -- /* -- * We must hold a refcount of the probed module while updating -- * its code to prohibit unexpected unloading. -- */ -- if (unlikely(!try_module_get(*probed_mod))) { -- ret = -ENOENT; -- goto out; -- } -- - /* - * If the module freed '.init.text', we couldn't insert - * kprobes in there. -@@ -1606,13 +1607,11 @@ static int check_kprobe_address_safe(struct kprobe *p, - if (within_module_init((unsigned long)p->addr, *probed_mod) && - !module_is_coming(*probed_mod)) { - module_put(*probed_mod); -- *probed_mod = NULL; - ret = -ENOENT; - } - } - - out: -- preempt_enable(); - jump_label_unlock(); - - return ret; diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c -index 19813b387ef9..2e07cb99cd4e 100644 +index 8857c0709bdd..e764c2f367f0 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c -@@ -5206,9 +5206,10 @@ static void scx_dump_task(struct seq_buf *s, struct scx_dump_ctx *dctx, +@@ -5277,9 +5277,10 @@ static void scx_dump_task(struct seq_buf *s, struct scx_dump_ctx *dctx, scx_get_task_state(p), p->scx.flags & ~SCX_TASK_STATE_MASK, p->scx.dsq_flags, ops_state & SCX_OPSS_STATE_MASK, ops_state >> SCX_OPSS_QSEQ_SHIFT); -- dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s dsq_vtime=%llu", +- dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s dsq_vtime=%llu slice=%llu", - p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf, -- p->scx.dsq_vtime); +- p->scx.dsq_vtime, p->scx.slice); + dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s", + p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf); + dump_line(s, " dsq_vtime=%llu slice=%llu weight=%u", @@ -15192,10 +12716,10 @@ index 19813b387ef9..2e07cb99cd4e 100644 if (SCX_HAS_OP(dump_task)) { diff --git a/scripts/package/PKGBUILD b/scripts/package/PKGBUILD -index dca706617adc..89d3aef160b7 100644 +index 0cf3a55b05e1..a27d4344a4e8 100644 --- a/scripts/package/PKGBUILD +++ b/scripts/package/PKGBUILD -@@ -91,6 +91,11 @@ _package-headers() { +@@ -90,6 +90,11 @@ _package-headers() { "${srctree}/scripts/package/install-extmod-build" "${builddir}" fi @@ -15208,15127 +12732,14 @@ index dca706617adc..89d3aef160b7 100644 mkdir -p "${builddir}" cp System.map "${builddir}/System.map" -- -2.48.0.rc1 +2.48.1 -From 775ce47a87b04e672a3cc3dbf8eb3e1f8dbc448a Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Sun, 2 Feb 2025 13:54:31 +0100 -Subject: [PATCH 07/12] itmt-core-ranking +From e31198eb6f1e2bc27e1bd80f6ed0633fc6b0a9fd Mon Sep 17 00:00:00 2001 +From: Eric Naim +Date: Mon, 3 Feb 2025 11:05:39 +0800 +Subject: [PATCH 6/6] zstd -Signed-off-by: Peter Jung ---- - arch/x86/include/asm/topology.h | 4 +- - arch/x86/kernel/itmt.c | 81 ++++++++++++++------------------- - arch/x86/kernel/smpboot.c | 19 +------- - kernel/sched/fair.c | 42 +++++++++++++---- - kernel/sched/sched.h | 1 - - kernel/sched/topology.c | 15 +----- - 6 files changed, 70 insertions(+), 92 deletions(-) - -diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h -index fd41103ad342..63bab25a4896 100644 ---- a/arch/x86/include/asm/topology.h -+++ b/arch/x86/include/asm/topology.h -@@ -250,7 +250,7 @@ extern bool x86_topology_update; - #include - - DECLARE_PER_CPU_READ_MOSTLY(int, sched_core_priority); --extern unsigned int __read_mostly sysctl_sched_itmt_enabled; -+extern bool __read_mostly sysctl_sched_itmt_enabled; - - /* Interface to set priority of a cpu */ - void sched_set_itmt_core_prio(int prio, int core_cpu); -@@ -263,7 +263,7 @@ void sched_clear_itmt_support(void); - - #else /* CONFIG_SCHED_MC_PRIO */ - --#define sysctl_sched_itmt_enabled 0 -+#define sysctl_sched_itmt_enabled false - static inline void sched_set_itmt_core_prio(int prio, int core_cpu) - { - } -diff --git a/arch/x86/kernel/itmt.c b/arch/x86/kernel/itmt.c -index 51b805c727fc..9cea1fc36c18 100644 ---- a/arch/x86/kernel/itmt.c -+++ b/arch/x86/kernel/itmt.c -@@ -19,6 +19,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -34,49 +35,38 @@ static bool __read_mostly sched_itmt_capable; - * of higher turbo frequency for cpus supporting Intel Turbo Boost Max - * Technology 3.0. - * -- * It can be set via /proc/sys/kernel/sched_itmt_enabled -+ * It can be set via /sys/kernel/debug/x86/sched_itmt_enabled - */ --unsigned int __read_mostly sysctl_sched_itmt_enabled; -+bool __read_mostly sysctl_sched_itmt_enabled; - --static int sched_itmt_update_handler(const struct ctl_table *table, int write, -- void *buffer, size_t *lenp, loff_t *ppos) -+static ssize_t sched_itmt_enabled_write(struct file *filp, -+ const char __user *ubuf, -+ size_t cnt, loff_t *ppos) - { -- unsigned int old_sysctl; -- int ret; -+ ssize_t result; -+ bool orig; - -- mutex_lock(&itmt_update_mutex); -+ guard(mutex)(&itmt_update_mutex); - -- if (!sched_itmt_capable) { -- mutex_unlock(&itmt_update_mutex); -- return -EINVAL; -- } -- -- old_sysctl = sysctl_sched_itmt_enabled; -- ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); -+ orig = sysctl_sched_itmt_enabled; -+ result = debugfs_write_file_bool(filp, ubuf, cnt, ppos); - -- if (!ret && write && old_sysctl != sysctl_sched_itmt_enabled) { -+ if (sysctl_sched_itmt_enabled != orig) { - x86_topology_update = true; - rebuild_sched_domains(); - } - -- mutex_unlock(&itmt_update_mutex); -- -- return ret; -+ return result; - } - --static struct ctl_table itmt_kern_table[] = { -- { -- .procname = "sched_itmt_enabled", -- .data = &sysctl_sched_itmt_enabled, -- .maxlen = sizeof(unsigned int), -- .mode = 0644, -- .proc_handler = sched_itmt_update_handler, -- .extra1 = SYSCTL_ZERO, -- .extra2 = SYSCTL_ONE, -- }, -+static const struct file_operations dfs_sched_itmt_fops = { -+ .read = debugfs_read_file_bool, -+ .write = sched_itmt_enabled_write, -+ .open = simple_open, -+ .llseek = default_llseek, - }; - --static struct ctl_table_header *itmt_sysctl_header; -+static struct dentry *dfs_sched_itmt; - - /** - * sched_set_itmt_support() - Indicate platform supports ITMT -@@ -97,16 +87,18 @@ static struct ctl_table_header *itmt_sysctl_header; - */ - int sched_set_itmt_support(void) - { -- mutex_lock(&itmt_update_mutex); -+ guard(mutex)(&itmt_update_mutex); - -- if (sched_itmt_capable) { -- mutex_unlock(&itmt_update_mutex); -+ if (sched_itmt_capable) - return 0; -- } - -- itmt_sysctl_header = register_sysctl("kernel", itmt_kern_table); -- if (!itmt_sysctl_header) { -- mutex_unlock(&itmt_update_mutex); -+ dfs_sched_itmt = debugfs_create_file_unsafe("sched_itmt_enabled", -+ 0644, -+ arch_debugfs_dir, -+ &sysctl_sched_itmt_enabled, -+ &dfs_sched_itmt_fops); -+ if (IS_ERR_OR_NULL(dfs_sched_itmt)) { -+ dfs_sched_itmt = NULL; - return -ENOMEM; - } - -@@ -117,8 +109,6 @@ int sched_set_itmt_support(void) - x86_topology_update = true; - rebuild_sched_domains(); - -- mutex_unlock(&itmt_update_mutex); -- - return 0; - } - -@@ -134,18 +124,15 @@ int sched_set_itmt_support(void) - */ - void sched_clear_itmt_support(void) - { -- mutex_lock(&itmt_update_mutex); -+ guard(mutex)(&itmt_update_mutex); - -- if (!sched_itmt_capable) { -- mutex_unlock(&itmt_update_mutex); -+ if (!sched_itmt_capable) - return; -- } -+ - sched_itmt_capable = false; - -- if (itmt_sysctl_header) { -- unregister_sysctl_table(itmt_sysctl_header); -- itmt_sysctl_header = NULL; -- } -+ debugfs_remove(dfs_sched_itmt); -+ dfs_sched_itmt = NULL; - - if (sysctl_sched_itmt_enabled) { - /* disable sched_itmt if we are no longer ITMT capable */ -@@ -153,8 +140,6 @@ void sched_clear_itmt_support(void) - x86_topology_update = true; - rebuild_sched_domains(); - } -- -- mutex_unlock(&itmt_update_mutex); - } - - int arch_asym_cpu_priority(int cpu) -diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c -index b5a8f0891135..ef63b1c0b491 100644 ---- a/arch/x86/kernel/smpboot.c -+++ b/arch/x86/kernel/smpboot.c -@@ -482,12 +482,6 @@ static int x86_core_flags(void) - return cpu_core_flags() | x86_sched_itmt_flags(); - } - #endif --#ifdef CONFIG_SCHED_SMT --static int x86_smt_flags(void) --{ -- return cpu_smt_flags(); --} --#endif - #ifdef CONFIG_SCHED_CLUSTER - static int x86_cluster_flags(void) - { -@@ -495,15 +489,6 @@ static int x86_cluster_flags(void) - } - #endif - --static int x86_die_flags(void) --{ -- if (cpu_feature_enabled(X86_FEATURE_HYBRID_CPU) || -- cpu_feature_enabled(X86_FEATURE_AMD_HETEROGENEOUS_CORES)) -- return x86_sched_itmt_flags(); -- -- return 0; --} -- - /* - * Set if a package/die has multiple NUMA nodes inside. - * AMD Magny-Cours, Intel Cluster-on-Die, and Intel -@@ -519,7 +504,7 @@ static void __init build_sched_topology(void) - - #ifdef CONFIG_SCHED_SMT - x86_topology[i++] = (struct sched_domain_topology_level){ -- cpu_smt_mask, x86_smt_flags, SD_INIT_NAME(SMT) -+ cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) - }; - #endif - #ifdef CONFIG_SCHED_CLUSTER -@@ -539,7 +524,7 @@ static void __init build_sched_topology(void) - */ - if (!x86_has_numa_in_package) { - x86_topology[i++] = (struct sched_domain_topology_level){ -- cpu_cpu_mask, x86_die_flags, SD_INIT_NAME(PKG) -+ cpu_cpu_mask, x86_sched_itmt_flags, SD_INIT_NAME(PKG) - }; - } - -diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c -index 8c0f17a96d4f..c532ffb153b4 100644 ---- a/kernel/sched/fair.c -+++ b/kernel/sched/fair.c -@@ -9836,6 +9836,8 @@ struct sg_lb_stats { - unsigned int group_weight; - enum group_type group_type; - unsigned int group_asym_packing; /* Tasks should be moved to preferred CPU */ -+ unsigned int asym_prefer_cpu; /* Group CPU with highest asym priority */ -+ int highest_asym_prio; /* Asym priority of asym_prefer_cpu */ - unsigned int group_smt_balance; /* Task on busy SMT be moved */ - unsigned long group_misfit_task_load; /* A CPU has a task too big for its capacity */ - #ifdef CONFIG_NUMA_BALANCING -@@ -10165,7 +10167,7 @@ sched_group_asym(struct lb_env *env, struct sg_lb_stats *sgs, struct sched_group - (sgs->group_weight - sgs->idle_cpus != 1)) - return false; - -- return sched_asym(env->sd, env->dst_cpu, group->asym_prefer_cpu); -+ return sched_asym(env->sd, env->dst_cpu, sgs->asym_prefer_cpu); - } - - /* One group has more than one SMT CPU while the other group does not */ -@@ -10246,6 +10248,17 @@ sched_reduced_capacity(struct rq *rq, struct sched_domain *sd) - return check_cpu_capacity(rq, sd); - } - -+static inline void -+update_sg_pick_asym_prefer(struct sg_lb_stats *sgs, int cpu) -+{ -+ int asym_prio = arch_asym_cpu_priority(cpu); -+ -+ if (asym_prio > sgs->highest_asym_prio) { -+ sgs->asym_prefer_cpu = cpu; -+ sgs->highest_asym_prio = asym_prio; -+ } -+} -+ - /** - * update_sg_lb_stats - Update sched_group's statistics for load balancing. - * @env: The load balancing environment. -@@ -10262,11 +10275,13 @@ static inline void update_sg_lb_stats(struct lb_env *env, - bool *sg_overloaded, - bool *sg_overutilized) - { -- int i, nr_running, local_group; -+ int i, nr_running, local_group, sd_flags = env->sd->flags; -+ bool balancing_at_rd = !env->sd->parent; - - memset(sgs, 0, sizeof(*sgs)); - - local_group = group == sds->local; -+ sgs->highest_asym_prio = INT_MIN; - - for_each_cpu_and(i, sched_group_span(group), env->cpus) { - struct rq *rq = cpu_rq(i); -@@ -10280,16 +10295,12 @@ static inline void update_sg_lb_stats(struct lb_env *env, - nr_running = rq->nr_running; - sgs->sum_nr_running += nr_running; - -- if (nr_running > 1) -- *sg_overloaded = 1; -+ if (sd_flags & SD_ASYM_PACKING) -+ update_sg_pick_asym_prefer(sgs, i); - - if (cpu_overutilized(i)) - *sg_overutilized = 1; - --#ifdef CONFIG_NUMA_BALANCING -- sgs->nr_numa_running += rq->nr_numa_running; -- sgs->nr_preferred_running += rq->nr_preferred_running; --#endif - /* - * No need to call idle_cpu() if nr_running is not 0 - */ -@@ -10299,10 +10310,21 @@ static inline void update_sg_lb_stats(struct lb_env *env, - continue; - } - -+ /* Overload indicator is only updated at root domain */ -+ if (balancing_at_rd && nr_running > 1) -+ *sg_overloaded = 1; -+ -+#ifdef CONFIG_NUMA_BALANCING -+ /* Only fbq_classify_group() uses this to classify NUMA groups */ -+ if (sd_flags & SD_NUMA) { -+ sgs->nr_numa_running += rq->nr_numa_running; -+ sgs->nr_preferred_running += rq->nr_preferred_running; -+ } -+#endif - if (local_group) - continue; - -- if (env->sd->flags & SD_ASYM_CPUCAPACITY) { -+ if (sd_flags & SD_ASYM_CPUCAPACITY) { - /* Check for a misfit task on the cpu */ - if (sgs->group_misfit_task_load < rq->misfit_task_load) { - sgs->group_misfit_task_load = rq->misfit_task_load; -@@ -10397,7 +10419,7 @@ static bool update_sd_pick_busiest(struct lb_env *env, - - case group_asym_packing: - /* Prefer to move from lowest priority CPU's work */ -- return sched_asym_prefer(sds->busiest->asym_prefer_cpu, sg->asym_prefer_cpu); -+ return sched_asym_prefer(busiest->asym_prefer_cpu, sgs->asym_prefer_cpu); - - case group_misfit_task: - /* -diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h -index da653eba7884..dee2797009e3 100644 ---- a/kernel/sched/sched.h -+++ b/kernel/sched/sched.h -@@ -2056,7 +2056,6 @@ struct sched_group { - unsigned int group_weight; - unsigned int cores; - struct sched_group_capacity *sgc; -- int asym_prefer_cpu; /* CPU of highest priority in group */ - int flags; - - /* -diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c -index 9748a4c8d668..59b8157cb114 100644 ---- a/kernel/sched/topology.c -+++ b/kernel/sched/topology.c -@@ -1302,7 +1302,7 @@ static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) - WARN_ON(!sg); - - do { -- int cpu, cores = 0, max_cpu = -1; -+ int cpu, cores = 0; - - sg->group_weight = cpumask_weight(sched_group_span(sg)); - -@@ -1314,19 +1314,6 @@ static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) - #endif - } - sg->cores = cores; -- -- if (!(sd->flags & SD_ASYM_PACKING)) -- goto next; -- -- for_each_cpu(cpu, sched_group_span(sg)) { -- if (max_cpu < 0) -- max_cpu = cpu; -- else if (sched_asym_prefer(cpu, max_cpu)) -- max_cpu = cpu; -- } -- sg->asym_prefer_cpu = max_cpu; -- --next: - sg = sg->next; - } while (sg != sd->groups); - --- -2.48.0.rc1 - -From b70533fb4d77af574e32e3ab1e48756b99313cd9 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Sun, 2 Feb 2025 13:54:42 +0100 -Subject: [PATCH 08/12] ntsync - -Signed-off-by: Peter Jung ---- - Documentation/userspace-api/index.rst | 1 + - Documentation/userspace-api/ntsync.rst | 385 +++++ - MAINTAINERS | 9 + - drivers/misc/Kconfig | 1 - - drivers/misc/ntsync.c | 1000 +++++++++++- - include/uapi/linux/ntsync.h | 42 +- - tools/testing/selftests/Makefile | 1 + - .../selftests/drivers/ntsync/.gitignore | 1 + - .../testing/selftests/drivers/ntsync/Makefile | 7 + - tools/testing/selftests/drivers/ntsync/config | 1 + - .../testing/selftests/drivers/ntsync/ntsync.c | 1343 +++++++++++++++++ - 11 files changed, 2772 insertions(+), 19 deletions(-) - create mode 100644 Documentation/userspace-api/ntsync.rst - create mode 100644 tools/testing/selftests/drivers/ntsync/.gitignore - create mode 100644 tools/testing/selftests/drivers/ntsync/Makefile - create mode 100644 tools/testing/selftests/drivers/ntsync/config - create mode 100644 tools/testing/selftests/drivers/ntsync/ntsync.c - -diff --git a/Documentation/userspace-api/index.rst b/Documentation/userspace-api/index.rst -index 274cc7546efc..9c1b15cd89ab 100644 ---- a/Documentation/userspace-api/index.rst -+++ b/Documentation/userspace-api/index.rst -@@ -63,6 +63,7 @@ Everything else - vduse - futex2 - perf_ring_buffer -+ ntsync - - .. only:: subproject and html - -diff --git a/Documentation/userspace-api/ntsync.rst b/Documentation/userspace-api/ntsync.rst -new file mode 100644 -index 000000000000..25e7c4aef968 ---- /dev/null -+++ b/Documentation/userspace-api/ntsync.rst -@@ -0,0 +1,385 @@ -+=================================== -+NT synchronization primitive driver -+=================================== -+ -+This page documents the user-space API for the ntsync driver. -+ -+ntsync is a support driver for emulation of NT synchronization -+primitives by user-space NT emulators. It exists because implementation -+in user-space, using existing tools, cannot match Windows performance -+while offering accurate semantics. It is implemented entirely in -+software, and does not drive any hardware device. -+ -+This interface is meant as a compatibility tool only, and should not -+be used for general synchronization. Instead use generic, versatile -+interfaces such as futex(2) and poll(2). -+ -+Synchronization primitives -+========================== -+ -+The ntsync driver exposes three types of synchronization primitives: -+semaphores, mutexes, and events. -+ -+A semaphore holds a single volatile 32-bit counter, and a static 32-bit -+integer denoting the maximum value. It is considered signaled (that is, -+can be acquired without contention, or will wake up a waiting thread) -+when the counter is nonzero. The counter is decremented by one when a -+wait is satisfied. Both the initial and maximum count are established -+when the semaphore is created. -+ -+A mutex holds a volatile 32-bit recursion count, and a volatile 32-bit -+identifier denoting its owner. A mutex is considered signaled when its -+owner is zero (indicating that it is not owned). The recursion count is -+incremented when a wait is satisfied, and ownership is set to the given -+identifier. -+ -+A mutex also holds an internal flag denoting whether its previous owner -+has died; such a mutex is said to be abandoned. Owner death is not -+tracked automatically based on thread death, but rather must be -+communicated using ``NTSYNC_IOC_MUTEX_KILL``. An abandoned mutex is -+inherently considered unowned. -+ -+Except for the "unowned" semantics of zero, the actual value of the -+owner identifier is not interpreted by the ntsync driver at all. The -+intended use is to store a thread identifier; however, the ntsync -+driver does not actually validate that a calling thread provides -+consistent or unique identifiers. -+ -+An event is similar to a semaphore with a maximum count of one. It holds -+a volatile boolean state denoting whether it is signaled or not. There -+are two types of events, auto-reset and manual-reset. An auto-reset -+event is designaled when a wait is satisfied; a manual-reset event is -+not. The event type is specified when the event is created. -+ -+Unless specified otherwise, all operations on an object are atomic and -+totally ordered with respect to other operations on the same object. -+ -+Objects are represented by files. When all file descriptors to an -+object are closed, that object is deleted. -+ -+Char device -+=========== -+ -+The ntsync driver creates a single char device /dev/ntsync. Each file -+description opened on the device represents a unique instance intended -+to back an individual NT virtual machine. Objects created by one ntsync -+instance may only be used with other objects created by the same -+instance. -+ -+ioctl reference -+=============== -+ -+All operations on the device are done through ioctls. There are four -+structures used in ioctl calls:: -+ -+ struct ntsync_sem_args { -+ __u32 count; -+ __u32 max; -+ }; -+ -+ struct ntsync_mutex_args { -+ __u32 owner; -+ __u32 count; -+ }; -+ -+ struct ntsync_event_args { -+ __u32 signaled; -+ __u32 manual; -+ }; -+ -+ struct ntsync_wait_args { -+ __u64 timeout; -+ __u64 objs; -+ __u32 count; -+ __u32 owner; -+ __u32 index; -+ __u32 alert; -+ __u32 flags; -+ __u32 pad; -+ }; -+ -+Depending on the ioctl, members of the structure may be used as input, -+output, or not at all. -+ -+The ioctls on the device file are as follows: -+ -+.. c:macro:: NTSYNC_IOC_CREATE_SEM -+ -+ Create a semaphore object. Takes a pointer to struct -+ :c:type:`ntsync_sem_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``count`` -+ - Initial count of the semaphore. -+ * - ``max`` -+ - Maximum count of the semaphore. -+ -+ Fails with ``EINVAL`` if ``count`` is greater than ``max``. -+ On success, returns a file descriptor the created semaphore. -+ -+.. c:macro:: NTSYNC_IOC_CREATE_MUTEX -+ -+ Create a mutex object. Takes a pointer to struct -+ :c:type:`ntsync_mutex_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``count`` -+ - Initial recursion count of the mutex. -+ * - ``owner`` -+ - Initial owner of the mutex. -+ -+ If ``owner`` is nonzero and ``count`` is zero, or if ``owner`` is -+ zero and ``count`` is nonzero, the function fails with ``EINVAL``. -+ On success, returns a file descriptor the created mutex. -+ -+.. c:macro:: NTSYNC_IOC_CREATE_EVENT -+ -+ Create an event object. Takes a pointer to struct -+ :c:type:`ntsync_event_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``signaled`` -+ - If nonzero, the event is initially signaled, otherwise -+ nonsignaled. -+ * - ``manual`` -+ - If nonzero, the event is a manual-reset event, otherwise -+ auto-reset. -+ -+ On success, returns a file descriptor the created event. -+ -+The ioctls on the individual objects are as follows: -+ -+.. c:macro:: NTSYNC_IOC_SEM_POST -+ -+ Post to a semaphore object. Takes a pointer to a 32-bit integer, -+ which on input holds the count to be added to the semaphore, and on -+ output contains its previous count. -+ -+ If adding to the semaphore's current count would raise the latter -+ past the semaphore's maximum count, the ioctl fails with -+ ``EOVERFLOW`` and the semaphore is not affected. If raising the -+ semaphore's count causes it to become signaled, eligible threads -+ waiting on this semaphore will be woken and the semaphore's count -+ decremented appropriately. -+ -+.. c:macro:: NTSYNC_IOC_MUTEX_UNLOCK -+ -+ Release a mutex object. Takes a pointer to struct -+ :c:type:`ntsync_mutex_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``owner`` -+ - Specifies the owner trying to release this mutex. -+ * - ``count`` -+ - On output, contains the previous recursion count. -+ -+ If ``owner`` is zero, the ioctl fails with ``EINVAL``. If ``owner`` -+ is not the current owner of the mutex, the ioctl fails with -+ ``EPERM``. -+ -+ The mutex's count will be decremented by one. If decrementing the -+ mutex's count causes it to become zero, the mutex is marked as -+ unowned and signaled, and eligible threads waiting on it will be -+ woken as appropriate. -+ -+.. c:macro:: NTSYNC_IOC_SET_EVENT -+ -+ Signal an event object. Takes a pointer to a 32-bit integer, which on -+ output contains the previous state of the event. -+ -+ Eligible threads will be woken, and auto-reset events will be -+ designaled appropriately. -+ -+.. c:macro:: NTSYNC_IOC_RESET_EVENT -+ -+ Designal an event object. Takes a pointer to a 32-bit integer, which -+ on output contains the previous state of the event. -+ -+.. c:macro:: NTSYNC_IOC_PULSE_EVENT -+ -+ Wake threads waiting on an event object while leaving it in an -+ unsignaled state. Takes a pointer to a 32-bit integer, which on -+ output contains the previous state of the event. -+ -+ A pulse operation can be thought of as a set followed by a reset, -+ performed as a single atomic operation. If two threads are waiting on -+ an auto-reset event which is pulsed, only one will be woken. If two -+ threads are waiting a manual-reset event which is pulsed, both will -+ be woken. However, in both cases, the event will be unsignaled -+ afterwards, and a simultaneous read operation will always report the -+ event as unsignaled. -+ -+.. c:macro:: NTSYNC_IOC_READ_SEM -+ -+ Read the current state of a semaphore object. Takes a pointer to -+ struct :c:type:`ntsync_sem_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``count`` -+ - On output, contains the current count of the semaphore. -+ * - ``max`` -+ - On output, contains the maximum count of the semaphore. -+ -+.. c:macro:: NTSYNC_IOC_READ_MUTEX -+ -+ Read the current state of a mutex object. Takes a pointer to struct -+ :c:type:`ntsync_mutex_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``owner`` -+ - On output, contains the current owner of the mutex, or zero -+ if the mutex is not currently owned. -+ * - ``count`` -+ - On output, contains the current recursion count of the mutex. -+ -+ If the mutex is marked as abandoned, the function fails with -+ ``EOWNERDEAD``. In this case, ``count`` and ``owner`` are set to -+ zero. -+ -+.. c:macro:: NTSYNC_IOC_READ_EVENT -+ -+ Read the current state of an event object. Takes a pointer to struct -+ :c:type:`ntsync_event_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``signaled`` -+ - On output, contains the current state of the event. -+ * - ``manual`` -+ - On output, contains 1 if the event is a manual-reset event, -+ and 0 otherwise. -+ -+.. c:macro:: NTSYNC_IOC_KILL_OWNER -+ -+ Mark a mutex as unowned and abandoned if it is owned by the given -+ owner. Takes an input-only pointer to a 32-bit integer denoting the -+ owner. If the owner is zero, the ioctl fails with ``EINVAL``. If the -+ owner does not own the mutex, the function fails with ``EPERM``. -+ -+ Eligible threads waiting on the mutex will be woken as appropriate -+ (and such waits will fail with ``EOWNERDEAD``, as described below). -+ -+.. c:macro:: NTSYNC_IOC_WAIT_ANY -+ -+ Poll on any of a list of objects, atomically acquiring at most one. -+ Takes a pointer to struct :c:type:`ntsync_wait_args`, which is -+ used as follows: -+ -+ .. list-table:: -+ -+ * - ``timeout`` -+ - Absolute timeout in nanoseconds. If ``NTSYNC_WAIT_REALTIME`` -+ is set, the timeout is measured against the REALTIME clock; -+ otherwise it is measured against the MONOTONIC clock. If the -+ timeout is equal to or earlier than the current time, the -+ function returns immediately without sleeping. If ``timeout`` -+ is U64_MAX, the function will sleep until an object is -+ signaled, and will not fail with ``ETIMEDOUT``. -+ * - ``objs`` -+ - Pointer to an array of ``count`` file descriptors -+ (specified as an integer so that the structure has the same -+ size regardless of architecture). If any object is -+ invalid, the function fails with ``EINVAL``. -+ * - ``count`` -+ - Number of objects specified in the ``objs`` array. -+ If greater than ``NTSYNC_MAX_WAIT_COUNT``, the function fails -+ with ``EINVAL``. -+ * - ``owner`` -+ - Mutex owner identifier. If any object in ``objs`` is a mutex, -+ the ioctl will attempt to acquire that mutex on behalf of -+ ``owner``. If ``owner`` is zero, the ioctl fails with -+ ``EINVAL``. -+ * - ``index`` -+ - On success, contains the index (into ``objs``) of the object -+ which was signaled. If ``alert`` was signaled instead, -+ this contains ``count``. -+ * - ``alert`` -+ - Optional event object file descriptor. If nonzero, this -+ specifies an "alert" event object which, if signaled, will -+ terminate the wait. If nonzero, the identifier must point to a -+ valid event. -+ * - ``flags`` -+ - Zero or more flags. Currently the only flag is -+ ``NTSYNC_WAIT_REALTIME``, which causes the timeout to be -+ measured against the REALTIME clock instead of MONOTONIC. -+ * - ``pad`` -+ - Unused, must be set to zero. -+ -+ This function attempts to acquire one of the given objects. If unable -+ to do so, it sleeps until an object becomes signaled, subsequently -+ acquiring it, or the timeout expires. In the latter case the ioctl -+ fails with ``ETIMEDOUT``. The function only acquires one object, even -+ if multiple objects are signaled. -+ -+ A semaphore is considered to be signaled if its count is nonzero, and -+ is acquired by decrementing its count by one. A mutex is considered -+ to be signaled if it is unowned or if its owner matches the ``owner`` -+ argument, and is acquired by incrementing its recursion count by one -+ and setting its owner to the ``owner`` argument. An auto-reset event -+ is acquired by designaling it; a manual-reset event is not affected -+ by acquisition. -+ -+ Acquisition is atomic and totally ordered with respect to other -+ operations on the same object. If two wait operations (with different -+ ``owner`` identifiers) are queued on the same mutex, only one is -+ signaled. If two wait operations are queued on the same semaphore, -+ and a value of one is posted to it, only one is signaled. -+ -+ If an abandoned mutex is acquired, the ioctl fails with -+ ``EOWNERDEAD``. Although this is a failure return, the function may -+ otherwise be considered successful. The mutex is marked as owned by -+ the given owner (with a recursion count of 1) and as no longer -+ abandoned, and ``index`` is still set to the index of the mutex. -+ -+ The ``alert`` argument is an "extra" event which can terminate the -+ wait, independently of all other objects. -+ -+ It is valid to pass the same object more than once, including by -+ passing the same event in the ``objs`` array and in ``alert``. If a -+ wakeup occurs due to that object being signaled, ``index`` is set to -+ the lowest index corresponding to that object. -+ -+ The function may fail with ``EINTR`` if a signal is received. -+ -+.. c:macro:: NTSYNC_IOC_WAIT_ALL -+ -+ Poll on a list of objects, atomically acquiring all of them. Takes a -+ pointer to struct :c:type:`ntsync_wait_args`, which is used -+ identically to ``NTSYNC_IOC_WAIT_ANY``, except that ``index`` is -+ always filled with zero on success if not woken via alert. -+ -+ This function attempts to simultaneously acquire all of the given -+ objects. If unable to do so, it sleeps until all objects become -+ simultaneously signaled, subsequently acquiring them, or the timeout -+ expires. In the latter case the ioctl fails with ``ETIMEDOUT`` and no -+ objects are modified. -+ -+ Objects may become signaled and subsequently designaled (through -+ acquisition by other threads) while this thread is sleeping. Only -+ once all objects are simultaneously signaled does the ioctl acquire -+ them and return. The entire acquisition is atomic and totally ordered -+ with respect to other operations on any of the given objects. -+ -+ If an abandoned mutex is acquired, the ioctl fails with -+ ``EOWNERDEAD``. Similarly to ``NTSYNC_IOC_WAIT_ANY``, all objects are -+ nevertheless marked as acquired. Note that if multiple mutex objects -+ are specified, there is no way to know which were marked as -+ abandoned. -+ -+ As with "any" waits, the ``alert`` argument is an "extra" event which -+ can terminate the wait. Critically, however, an "all" wait will -+ succeed if all members in ``objs`` are signaled, *or* if ``alert`` is -+ signaled. In the latter case ``index`` will be set to ``count``. As -+ with "any" waits, if both conditions are filled, the former takes -+ priority, and objects in ``objs`` will be acquired. -+ -+ Unlike ``NTSYNC_IOC_WAIT_ANY``, it is not valid to pass the same -+ object more than once, nor is it valid to pass the same object in -+ ``objs`` and in ``alert``. If this is attempted, the function fails -+ with ``EINVAL``. -diff --git a/MAINTAINERS b/MAINTAINERS -index 0fa7c5728f1e..efecb59adfe6 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -16709,6 +16709,15 @@ T: git https://github.com/Paragon-Software-Group/linux-ntfs3.git - F: Documentation/filesystems/ntfs3.rst - F: fs/ntfs3/ - -+NTSYNC SYNCHRONIZATION PRIMITIVE DRIVER -+M: Elizabeth Figura -+L: wine-devel@winehq.org -+S: Supported -+F: Documentation/userspace-api/ntsync.rst -+F: drivers/misc/ntsync.c -+F: include/uapi/linux/ntsync.h -+F: tools/testing/selftests/drivers/ntsync/ -+ - NUBUS SUBSYSTEM - M: Finn Thain - L: linux-m68k@lists.linux-m68k.org -diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig -index 09cbe3f0ab1e..fb772bfe27c3 100644 ---- a/drivers/misc/Kconfig -+++ b/drivers/misc/Kconfig -@@ -517,7 +517,6 @@ config OPEN_DICE - - config NTSYNC - tristate "NT synchronization primitive emulation" -- depends on BROKEN - help - This module provides kernel support for emulation of Windows NT - synchronization primitives. It is not a hardware driver. -diff --git a/drivers/misc/ntsync.c b/drivers/misc/ntsync.c -index 4954553b7baa..055395cde42b 100644 ---- a/drivers/misc/ntsync.c -+++ b/drivers/misc/ntsync.c -@@ -6,11 +6,17 @@ - */ - - #include -+#include - #include - #include -+#include -+#include - #include - #include -+#include - #include -+#include -+#include - #include - #include - #include -@@ -19,6 +25,8 @@ - - enum ntsync_type { - NTSYNC_TYPE_SEM, -+ NTSYNC_TYPE_MUTEX, -+ NTSYNC_TYPE_EVENT, - }; - - /* -@@ -30,10 +38,13 @@ enum ntsync_type { - * - * Both rely on struct file for reference counting. Individual - * ntsync_obj objects take a reference to the device when created. -+ * Wait operations take a reference to each object being waited on for -+ * the duration of the wait. - */ - - struct ntsync_obj { - spinlock_t lock; -+ int dev_locked; - - enum ntsync_type type; - -@@ -46,22 +57,344 @@ struct ntsync_obj { - __u32 count; - __u32 max; - } sem; -+ struct { -+ __u32 count; -+ pid_t owner; -+ bool ownerdead; -+ } mutex; -+ struct { -+ bool manual; -+ bool signaled; -+ } event; - } u; -+ -+ /* -+ * any_waiters is protected by the object lock, but all_waiters is -+ * protected by the device wait_all_lock. -+ */ -+ struct list_head any_waiters; -+ struct list_head all_waiters; -+ -+ /* -+ * Hint describing how many tasks are queued on this object in a -+ * wait-all operation. -+ * -+ * Any time we do a wake, we may need to wake "all" waiters as well as -+ * "any" waiters. In order to atomically wake "all" waiters, we must -+ * lock all of the objects, and that means grabbing the wait_all_lock -+ * below (and, due to lock ordering rules, before locking this object). -+ * However, wait-all is a rare operation, and grabbing the wait-all -+ * lock for every wake would create unnecessary contention. -+ * Therefore we first check whether all_hint is zero, and, if it is, -+ * we skip trying to wake "all" waiters. -+ * -+ * Since wait requests must originate from user-space threads, we're -+ * limited here by PID_MAX_LIMIT, so there's no risk of overflow. -+ */ -+ atomic_t all_hint; -+}; -+ -+struct ntsync_q_entry { -+ struct list_head node; -+ struct ntsync_q *q; -+ struct ntsync_obj *obj; -+ __u32 index; -+}; -+ -+struct ntsync_q { -+ struct task_struct *task; -+ __u32 owner; -+ -+ /* -+ * Protected via atomic_try_cmpxchg(). Only the thread that wins the -+ * compare-and-swap may actually change object states and wake this -+ * task. -+ */ -+ atomic_t signaled; -+ -+ bool all; -+ bool ownerdead; -+ __u32 count; -+ struct ntsync_q_entry entries[]; - }; - - struct ntsync_device { -+ /* -+ * Wait-all operations must atomically grab all objects, and be totally -+ * ordered with respect to each other and wait-any operations. -+ * If one thread is trying to acquire several objects, another thread -+ * cannot touch the object at the same time. -+ * -+ * This device-wide lock is used to serialize wait-for-all -+ * operations, and operations on an object that is involved in a -+ * wait-for-all. -+ */ -+ struct mutex wait_all_lock; -+ - struct file *file; - }; - -+/* -+ * Single objects are locked using obj->lock. -+ * -+ * Multiple objects are 'locked' while holding dev->wait_all_lock. -+ * In this case however, individual objects are not locked by holding -+ * obj->lock, but by setting obj->dev_locked. -+ * -+ * This means that in order to lock a single object, the sequence is slightly -+ * more complicated than usual. Specifically it needs to check obj->dev_locked -+ * after acquiring obj->lock, if set, it needs to drop the lock and acquire -+ * dev->wait_all_lock in order to serialize against the multi-object operation. -+ */ -+ -+static void dev_lock_obj(struct ntsync_device *dev, struct ntsync_obj *obj) -+{ -+ lockdep_assert_held(&dev->wait_all_lock); -+ lockdep_assert(obj->dev == dev); -+ spin_lock(&obj->lock); -+ /* -+ * By setting obj->dev_locked inside obj->lock, it is ensured that -+ * anyone holding obj->lock must see the value. -+ */ -+ obj->dev_locked = 1; -+ spin_unlock(&obj->lock); -+} -+ -+static void dev_unlock_obj(struct ntsync_device *dev, struct ntsync_obj *obj) -+{ -+ lockdep_assert_held(&dev->wait_all_lock); -+ lockdep_assert(obj->dev == dev); -+ spin_lock(&obj->lock); -+ obj->dev_locked = 0; -+ spin_unlock(&obj->lock); -+} -+ -+static void obj_lock(struct ntsync_obj *obj) -+{ -+ struct ntsync_device *dev = obj->dev; -+ -+ for (;;) { -+ spin_lock(&obj->lock); -+ if (likely(!obj->dev_locked)) -+ break; -+ -+ spin_unlock(&obj->lock); -+ mutex_lock(&dev->wait_all_lock); -+ spin_lock(&obj->lock); -+ /* -+ * obj->dev_locked should be set and released under the same -+ * wait_all_lock section, since we now own this lock, it should -+ * be clear. -+ */ -+ lockdep_assert(!obj->dev_locked); -+ spin_unlock(&obj->lock); -+ mutex_unlock(&dev->wait_all_lock); -+ } -+} -+ -+static void obj_unlock(struct ntsync_obj *obj) -+{ -+ spin_unlock(&obj->lock); -+} -+ -+static bool ntsync_lock_obj(struct ntsync_device *dev, struct ntsync_obj *obj) -+{ -+ bool all; -+ -+ obj_lock(obj); -+ all = atomic_read(&obj->all_hint); -+ if (unlikely(all)) { -+ obj_unlock(obj); -+ mutex_lock(&dev->wait_all_lock); -+ dev_lock_obj(dev, obj); -+ } -+ -+ return all; -+} -+ -+static void ntsync_unlock_obj(struct ntsync_device *dev, struct ntsync_obj *obj, bool all) -+{ -+ if (all) { -+ dev_unlock_obj(dev, obj); -+ mutex_unlock(&dev->wait_all_lock); -+ } else { -+ obj_unlock(obj); -+ } -+} -+ -+#define ntsync_assert_held(obj) \ -+ lockdep_assert((lockdep_is_held(&(obj)->lock) != LOCK_STATE_NOT_HELD) || \ -+ ((lockdep_is_held(&(obj)->dev->wait_all_lock) != LOCK_STATE_NOT_HELD) && \ -+ (obj)->dev_locked)) -+ -+static bool is_signaled(struct ntsync_obj *obj, __u32 owner) -+{ -+ ntsync_assert_held(obj); -+ -+ switch (obj->type) { -+ case NTSYNC_TYPE_SEM: -+ return !!obj->u.sem.count; -+ case NTSYNC_TYPE_MUTEX: -+ if (obj->u.mutex.owner && obj->u.mutex.owner != owner) -+ return false; -+ return obj->u.mutex.count < UINT_MAX; -+ case NTSYNC_TYPE_EVENT: -+ return obj->u.event.signaled; -+ } -+ -+ WARN(1, "bad object type %#x\n", obj->type); -+ return false; -+} -+ -+/* -+ * "locked_obj" is an optional pointer to an object which is already locked and -+ * should not be locked again. This is necessary so that changing an object's -+ * state and waking it can be a single atomic operation. -+ */ -+static void try_wake_all(struct ntsync_device *dev, struct ntsync_q *q, -+ struct ntsync_obj *locked_obj) -+{ -+ __u32 count = q->count; -+ bool can_wake = true; -+ int signaled = -1; -+ __u32 i; -+ -+ lockdep_assert_held(&dev->wait_all_lock); -+ if (locked_obj) -+ lockdep_assert(locked_obj->dev_locked); -+ -+ for (i = 0; i < count; i++) { -+ if (q->entries[i].obj != locked_obj) -+ dev_lock_obj(dev, q->entries[i].obj); -+ } -+ -+ for (i = 0; i < count; i++) { -+ if (!is_signaled(q->entries[i].obj, q->owner)) { -+ can_wake = false; -+ break; -+ } -+ } -+ -+ if (can_wake && atomic_try_cmpxchg(&q->signaled, &signaled, 0)) { -+ for (i = 0; i < count; i++) { -+ struct ntsync_obj *obj = q->entries[i].obj; -+ -+ switch (obj->type) { -+ case NTSYNC_TYPE_SEM: -+ obj->u.sem.count--; -+ break; -+ case NTSYNC_TYPE_MUTEX: -+ if (obj->u.mutex.ownerdead) -+ q->ownerdead = true; -+ obj->u.mutex.ownerdead = false; -+ obj->u.mutex.count++; -+ obj->u.mutex.owner = q->owner; -+ break; -+ case NTSYNC_TYPE_EVENT: -+ if (!obj->u.event.manual) -+ obj->u.event.signaled = false; -+ break; -+ } -+ } -+ wake_up_process(q->task); -+ } -+ -+ for (i = 0; i < count; i++) { -+ if (q->entries[i].obj != locked_obj) -+ dev_unlock_obj(dev, q->entries[i].obj); -+ } -+} -+ -+static void try_wake_all_obj(struct ntsync_device *dev, struct ntsync_obj *obj) -+{ -+ struct ntsync_q_entry *entry; -+ -+ lockdep_assert_held(&dev->wait_all_lock); -+ lockdep_assert(obj->dev_locked); -+ -+ list_for_each_entry(entry, &obj->all_waiters, node) -+ try_wake_all(dev, entry->q, obj); -+} -+ -+static void try_wake_any_sem(struct ntsync_obj *sem) -+{ -+ struct ntsync_q_entry *entry; -+ -+ ntsync_assert_held(sem); -+ lockdep_assert(sem->type == NTSYNC_TYPE_SEM); -+ -+ list_for_each_entry(entry, &sem->any_waiters, node) { -+ struct ntsync_q *q = entry->q; -+ int signaled = -1; -+ -+ if (!sem->u.sem.count) -+ break; -+ -+ if (atomic_try_cmpxchg(&q->signaled, &signaled, entry->index)) { -+ sem->u.sem.count--; -+ wake_up_process(q->task); -+ } -+ } -+} -+ -+static void try_wake_any_mutex(struct ntsync_obj *mutex) -+{ -+ struct ntsync_q_entry *entry; -+ -+ ntsync_assert_held(mutex); -+ lockdep_assert(mutex->type == NTSYNC_TYPE_MUTEX); -+ -+ list_for_each_entry(entry, &mutex->any_waiters, node) { -+ struct ntsync_q *q = entry->q; -+ int signaled = -1; -+ -+ if (mutex->u.mutex.count == UINT_MAX) -+ break; -+ if (mutex->u.mutex.owner && mutex->u.mutex.owner != q->owner) -+ continue; -+ -+ if (atomic_try_cmpxchg(&q->signaled, &signaled, entry->index)) { -+ if (mutex->u.mutex.ownerdead) -+ q->ownerdead = true; -+ mutex->u.mutex.ownerdead = false; -+ mutex->u.mutex.count++; -+ mutex->u.mutex.owner = q->owner; -+ wake_up_process(q->task); -+ } -+ } -+} -+ -+static void try_wake_any_event(struct ntsync_obj *event) -+{ -+ struct ntsync_q_entry *entry; -+ -+ ntsync_assert_held(event); -+ lockdep_assert(event->type == NTSYNC_TYPE_EVENT); -+ -+ list_for_each_entry(entry, &event->any_waiters, node) { -+ struct ntsync_q *q = entry->q; -+ int signaled = -1; -+ -+ if (!event->u.event.signaled) -+ break; -+ -+ if (atomic_try_cmpxchg(&q->signaled, &signaled, entry->index)) { -+ if (!event->u.event.manual) -+ event->u.event.signaled = false; -+ wake_up_process(q->task); -+ } -+ } -+} -+ - /* - * Actually change the semaphore state, returning -EOVERFLOW if it is made - * invalid. - */ --static int post_sem_state(struct ntsync_obj *sem, __u32 count) -+static int release_sem_state(struct ntsync_obj *sem, __u32 count) - { - __u32 sum; - -- lockdep_assert_held(&sem->lock); -+ ntsync_assert_held(sem); - - if (check_add_overflow(sem->u.sem.count, count, &sum) || - sum > sem->u.sem.max) -@@ -71,11 +404,13 @@ static int post_sem_state(struct ntsync_obj *sem, __u32 count) - return 0; - } - --static int ntsync_sem_post(struct ntsync_obj *sem, void __user *argp) -+static int ntsync_sem_release(struct ntsync_obj *sem, void __user *argp) - { -+ struct ntsync_device *dev = sem->dev; - __u32 __user *user_args = argp; - __u32 prev_count; - __u32 args; -+ bool all; - int ret; - - if (copy_from_user(&args, argp, sizeof(args))) -@@ -84,12 +419,17 @@ static int ntsync_sem_post(struct ntsync_obj *sem, void __user *argp) - if (sem->type != NTSYNC_TYPE_SEM) - return -EINVAL; - -- spin_lock(&sem->lock); -+ all = ntsync_lock_obj(dev, sem); - - prev_count = sem->u.sem.count; -- ret = post_sem_state(sem, args); -+ ret = release_sem_state(sem, args); -+ if (!ret) { -+ if (all) -+ try_wake_all_obj(dev, sem); -+ try_wake_any_sem(sem); -+ } - -- spin_unlock(&sem->lock); -+ ntsync_unlock_obj(dev, sem, all); - - if (!ret && put_user(prev_count, user_args)) - ret = -EFAULT; -@@ -97,13 +437,229 @@ static int ntsync_sem_post(struct ntsync_obj *sem, void __user *argp) - return ret; - } - --static int ntsync_obj_release(struct inode *inode, struct file *file) -+/* -+ * Actually change the mutex state, returning -EPERM if not the owner. -+ */ -+static int unlock_mutex_state(struct ntsync_obj *mutex, -+ const struct ntsync_mutex_args *args) - { -- struct ntsync_obj *obj = file->private_data; -+ ntsync_assert_held(mutex); -+ -+ if (mutex->u.mutex.owner != args->owner) -+ return -EPERM; -+ -+ if (!--mutex->u.mutex.count) -+ mutex->u.mutex.owner = 0; -+ return 0; -+} -+ -+static int ntsync_mutex_unlock(struct ntsync_obj *mutex, void __user *argp) -+{ -+ struct ntsync_mutex_args __user *user_args = argp; -+ struct ntsync_device *dev = mutex->dev; -+ struct ntsync_mutex_args args; -+ __u32 prev_count; -+ bool all; -+ int ret; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ if (!args.owner) -+ return -EINVAL; -+ -+ if (mutex->type != NTSYNC_TYPE_MUTEX) -+ return -EINVAL; -+ -+ all = ntsync_lock_obj(dev, mutex); -+ -+ prev_count = mutex->u.mutex.count; -+ ret = unlock_mutex_state(mutex, &args); -+ if (!ret) { -+ if (all) -+ try_wake_all_obj(dev, mutex); -+ try_wake_any_mutex(mutex); -+ } -+ -+ ntsync_unlock_obj(dev, mutex, all); -+ -+ if (!ret && put_user(prev_count, &user_args->count)) -+ ret = -EFAULT; -+ -+ return ret; -+} -+ -+/* -+ * Actually change the mutex state to mark its owner as dead, -+ * returning -EPERM if not the owner. -+ */ -+static int kill_mutex_state(struct ntsync_obj *mutex, __u32 owner) -+{ -+ ntsync_assert_held(mutex); -+ -+ if (mutex->u.mutex.owner != owner) -+ return -EPERM; -+ -+ mutex->u.mutex.ownerdead = true; -+ mutex->u.mutex.owner = 0; -+ mutex->u.mutex.count = 0; -+ return 0; -+} -+ -+static int ntsync_mutex_kill(struct ntsync_obj *mutex, void __user *argp) -+{ -+ struct ntsync_device *dev = mutex->dev; -+ __u32 owner; -+ bool all; -+ int ret; -+ -+ if (get_user(owner, (__u32 __user *)argp)) -+ return -EFAULT; -+ if (!owner) -+ return -EINVAL; -+ -+ if (mutex->type != NTSYNC_TYPE_MUTEX) -+ return -EINVAL; -+ -+ all = ntsync_lock_obj(dev, mutex); -+ -+ ret = kill_mutex_state(mutex, owner); -+ if (!ret) { -+ if (all) -+ try_wake_all_obj(dev, mutex); -+ try_wake_any_mutex(mutex); -+ } -+ -+ ntsync_unlock_obj(dev, mutex, all); -+ -+ return ret; -+} -+ -+static int ntsync_event_set(struct ntsync_obj *event, void __user *argp, bool pulse) -+{ -+ struct ntsync_device *dev = event->dev; -+ __u32 prev_state; -+ bool all; -+ -+ if (event->type != NTSYNC_TYPE_EVENT) -+ return -EINVAL; -+ -+ all = ntsync_lock_obj(dev, event); -+ -+ prev_state = event->u.event.signaled; -+ event->u.event.signaled = true; -+ if (all) -+ try_wake_all_obj(dev, event); -+ try_wake_any_event(event); -+ if (pulse) -+ event->u.event.signaled = false; -+ -+ ntsync_unlock_obj(dev, event, all); -+ -+ if (put_user(prev_state, (__u32 __user *)argp)) -+ return -EFAULT; -+ -+ return 0; -+} -+ -+static int ntsync_event_reset(struct ntsync_obj *event, void __user *argp) -+{ -+ struct ntsync_device *dev = event->dev; -+ __u32 prev_state; -+ bool all; -+ -+ if (event->type != NTSYNC_TYPE_EVENT) -+ return -EINVAL; -+ -+ all = ntsync_lock_obj(dev, event); -+ -+ prev_state = event->u.event.signaled; -+ event->u.event.signaled = false; -+ -+ ntsync_unlock_obj(dev, event, all); -+ -+ if (put_user(prev_state, (__u32 __user *)argp)) -+ return -EFAULT; -+ -+ return 0; -+} -+ -+static int ntsync_sem_read(struct ntsync_obj *sem, void __user *argp) -+{ -+ struct ntsync_sem_args __user *user_args = argp; -+ struct ntsync_device *dev = sem->dev; -+ struct ntsync_sem_args args; -+ bool all; -+ -+ if (sem->type != NTSYNC_TYPE_SEM) -+ return -EINVAL; -+ -+ all = ntsync_lock_obj(dev, sem); -+ -+ args.count = sem->u.sem.count; -+ args.max = sem->u.sem.max; -+ -+ ntsync_unlock_obj(dev, sem, all); -+ -+ if (copy_to_user(user_args, &args, sizeof(args))) -+ return -EFAULT; -+ return 0; -+} - -+static int ntsync_mutex_read(struct ntsync_obj *mutex, void __user *argp) -+{ -+ struct ntsync_mutex_args __user *user_args = argp; -+ struct ntsync_device *dev = mutex->dev; -+ struct ntsync_mutex_args args; -+ bool all; -+ int ret; -+ -+ if (mutex->type != NTSYNC_TYPE_MUTEX) -+ return -EINVAL; -+ -+ all = ntsync_lock_obj(dev, mutex); -+ -+ args.count = mutex->u.mutex.count; -+ args.owner = mutex->u.mutex.owner; -+ ret = mutex->u.mutex.ownerdead ? -EOWNERDEAD : 0; -+ -+ ntsync_unlock_obj(dev, mutex, all); -+ -+ if (copy_to_user(user_args, &args, sizeof(args))) -+ return -EFAULT; -+ return ret; -+} -+ -+static int ntsync_event_read(struct ntsync_obj *event, void __user *argp) -+{ -+ struct ntsync_event_args __user *user_args = argp; -+ struct ntsync_device *dev = event->dev; -+ struct ntsync_event_args args; -+ bool all; -+ -+ if (event->type != NTSYNC_TYPE_EVENT) -+ return -EINVAL; -+ -+ all = ntsync_lock_obj(dev, event); -+ -+ args.manual = event->u.event.manual; -+ args.signaled = event->u.event.signaled; -+ -+ ntsync_unlock_obj(dev, event, all); -+ -+ if (copy_to_user(user_args, &args, sizeof(args))) -+ return -EFAULT; -+ return 0; -+} -+ -+static void ntsync_free_obj(struct ntsync_obj *obj) -+{ - fput(obj->dev->file); - kfree(obj); -+} - -+static int ntsync_obj_release(struct inode *inode, struct file *file) -+{ -+ ntsync_free_obj(file->private_data); - return 0; - } - -@@ -114,8 +670,24 @@ static long ntsync_obj_ioctl(struct file *file, unsigned int cmd, - void __user *argp = (void __user *)parm; - - switch (cmd) { -- case NTSYNC_IOC_SEM_POST: -- return ntsync_sem_post(obj, argp); -+ case NTSYNC_IOC_SEM_RELEASE: -+ return ntsync_sem_release(obj, argp); -+ case NTSYNC_IOC_SEM_READ: -+ return ntsync_sem_read(obj, argp); -+ case NTSYNC_IOC_MUTEX_UNLOCK: -+ return ntsync_mutex_unlock(obj, argp); -+ case NTSYNC_IOC_MUTEX_KILL: -+ return ntsync_mutex_kill(obj, argp); -+ case NTSYNC_IOC_MUTEX_READ: -+ return ntsync_mutex_read(obj, argp); -+ case NTSYNC_IOC_EVENT_SET: -+ return ntsync_event_set(obj, argp, false); -+ case NTSYNC_IOC_EVENT_RESET: -+ return ntsync_event_reset(obj, argp); -+ case NTSYNC_IOC_EVENT_PULSE: -+ return ntsync_event_set(obj, argp, true); -+ case NTSYNC_IOC_EVENT_READ: -+ return ntsync_event_read(obj, argp); - default: - return -ENOIOCTLCMD; - } -@@ -140,6 +712,9 @@ static struct ntsync_obj *ntsync_alloc_obj(struct ntsync_device *dev, - obj->dev = dev; - get_file(dev->file); - spin_lock_init(&obj->lock); -+ INIT_LIST_HEAD(&obj->any_waiters); -+ INIT_LIST_HEAD(&obj->all_waiters); -+ atomic_set(&obj->all_hint, 0); - - return obj; - } -@@ -165,7 +740,6 @@ static int ntsync_obj_get_fd(struct ntsync_obj *obj) - - static int ntsync_create_sem(struct ntsync_device *dev, void __user *argp) - { -- struct ntsync_sem_args __user *user_args = argp; - struct ntsync_sem_args args; - struct ntsync_obj *sem; - int fd; -@@ -182,12 +756,398 @@ static int ntsync_create_sem(struct ntsync_device *dev, void __user *argp) - sem->u.sem.count = args.count; - sem->u.sem.max = args.max; - fd = ntsync_obj_get_fd(sem); -- if (fd < 0) { -- kfree(sem); -- return fd; -+ if (fd < 0) -+ ntsync_free_obj(sem); -+ -+ return fd; -+} -+ -+static int ntsync_create_mutex(struct ntsync_device *dev, void __user *argp) -+{ -+ struct ntsync_mutex_args args; -+ struct ntsync_obj *mutex; -+ int fd; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ if (!args.owner != !args.count) -+ return -EINVAL; -+ -+ mutex = ntsync_alloc_obj(dev, NTSYNC_TYPE_MUTEX); -+ if (!mutex) -+ return -ENOMEM; -+ mutex->u.mutex.count = args.count; -+ mutex->u.mutex.owner = args.owner; -+ fd = ntsync_obj_get_fd(mutex); -+ if (fd < 0) -+ ntsync_free_obj(mutex); -+ -+ return fd; -+} -+ -+static int ntsync_create_event(struct ntsync_device *dev, void __user *argp) -+{ -+ struct ntsync_event_args args; -+ struct ntsync_obj *event; -+ int fd; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ event = ntsync_alloc_obj(dev, NTSYNC_TYPE_EVENT); -+ if (!event) -+ return -ENOMEM; -+ event->u.event.manual = args.manual; -+ event->u.event.signaled = args.signaled; -+ fd = ntsync_obj_get_fd(event); -+ if (fd < 0) -+ ntsync_free_obj(event); -+ -+ return fd; -+} -+ -+static struct ntsync_obj *get_obj(struct ntsync_device *dev, int fd) -+{ -+ struct file *file = fget(fd); -+ struct ntsync_obj *obj; -+ -+ if (!file) -+ return NULL; -+ -+ if (file->f_op != &ntsync_obj_fops) { -+ fput(file); -+ return NULL; -+ } -+ -+ obj = file->private_data; -+ if (obj->dev != dev) { -+ fput(file); -+ return NULL; - } - -- return put_user(fd, &user_args->sem); -+ return obj; -+} -+ -+static void put_obj(struct ntsync_obj *obj) -+{ -+ fput(obj->file); -+} -+ -+static int ntsync_schedule(const struct ntsync_q *q, const struct ntsync_wait_args *args) -+{ -+ ktime_t timeout = ns_to_ktime(args->timeout); -+ clockid_t clock = CLOCK_MONOTONIC; -+ ktime_t *timeout_ptr; -+ int ret = 0; -+ -+ timeout_ptr = (args->timeout == U64_MAX ? NULL : &timeout); -+ -+ if (args->flags & NTSYNC_WAIT_REALTIME) -+ clock = CLOCK_REALTIME; -+ -+ do { -+ if (signal_pending(current)) { -+ ret = -ERESTARTSYS; -+ break; -+ } -+ -+ set_current_state(TASK_INTERRUPTIBLE); -+ if (atomic_read(&q->signaled) != -1) { -+ ret = 0; -+ break; -+ } -+ ret = schedule_hrtimeout_range_clock(timeout_ptr, 0, HRTIMER_MODE_ABS, clock); -+ } while (ret < 0); -+ __set_current_state(TASK_RUNNING); -+ -+ return ret; -+} -+ -+/* -+ * Allocate and initialize the ntsync_q structure, but do not queue us yet. -+ */ -+static int setup_wait(struct ntsync_device *dev, -+ const struct ntsync_wait_args *args, bool all, -+ struct ntsync_q **ret_q) -+{ -+ int fds[NTSYNC_MAX_WAIT_COUNT + 1]; -+ const __u32 count = args->count; -+ struct ntsync_q *q; -+ __u32 total_count; -+ __u32 i, j; -+ -+ if (args->pad || (args->flags & ~NTSYNC_WAIT_REALTIME)) -+ return -EINVAL; -+ -+ if (args->count > NTSYNC_MAX_WAIT_COUNT) -+ return -EINVAL; -+ -+ total_count = count; -+ if (args->alert) -+ total_count++; -+ -+ if (copy_from_user(fds, u64_to_user_ptr(args->objs), -+ array_size(count, sizeof(*fds)))) -+ return -EFAULT; -+ if (args->alert) -+ fds[count] = args->alert; -+ -+ q = kmalloc(struct_size(q, entries, total_count), GFP_KERNEL); -+ if (!q) -+ return -ENOMEM; -+ q->task = current; -+ q->owner = args->owner; -+ atomic_set(&q->signaled, -1); -+ q->all = all; -+ q->ownerdead = false; -+ q->count = count; -+ -+ for (i = 0; i < total_count; i++) { -+ struct ntsync_q_entry *entry = &q->entries[i]; -+ struct ntsync_obj *obj = get_obj(dev, fds[i]); -+ -+ if (!obj) -+ goto err; -+ -+ if (all) { -+ /* Check that the objects are all distinct. */ -+ for (j = 0; j < i; j++) { -+ if (obj == q->entries[j].obj) { -+ put_obj(obj); -+ goto err; -+ } -+ } -+ } -+ -+ entry->obj = obj; -+ entry->q = q; -+ entry->index = i; -+ } -+ -+ *ret_q = q; -+ return 0; -+ -+err: -+ for (j = 0; j < i; j++) -+ put_obj(q->entries[j].obj); -+ kfree(q); -+ return -EINVAL; -+} -+ -+static void try_wake_any_obj(struct ntsync_obj *obj) -+{ -+ switch (obj->type) { -+ case NTSYNC_TYPE_SEM: -+ try_wake_any_sem(obj); -+ break; -+ case NTSYNC_TYPE_MUTEX: -+ try_wake_any_mutex(obj); -+ break; -+ case NTSYNC_TYPE_EVENT: -+ try_wake_any_event(obj); -+ break; -+ } -+} -+ -+static int ntsync_wait_any(struct ntsync_device *dev, void __user *argp) -+{ -+ struct ntsync_wait_args args; -+ __u32 i, total_count; -+ struct ntsync_q *q; -+ int signaled; -+ bool all; -+ int ret; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ ret = setup_wait(dev, &args, false, &q); -+ if (ret < 0) -+ return ret; -+ -+ total_count = args.count; -+ if (args.alert) -+ total_count++; -+ -+ /* queue ourselves */ -+ -+ for (i = 0; i < total_count; i++) { -+ struct ntsync_q_entry *entry = &q->entries[i]; -+ struct ntsync_obj *obj = entry->obj; -+ -+ all = ntsync_lock_obj(dev, obj); -+ list_add_tail(&entry->node, &obj->any_waiters); -+ ntsync_unlock_obj(dev, obj, all); -+ } -+ -+ /* -+ * Check if we are already signaled. -+ * -+ * Note that the API requires that normal objects are checked before -+ * the alert event. Hence we queue the alert event last, and check -+ * objects in order. -+ */ -+ -+ for (i = 0; i < total_count; i++) { -+ struct ntsync_obj *obj = q->entries[i].obj; -+ -+ if (atomic_read(&q->signaled) != -1) -+ break; -+ -+ all = ntsync_lock_obj(dev, obj); -+ try_wake_any_obj(obj); -+ ntsync_unlock_obj(dev, obj, all); -+ } -+ -+ /* sleep */ -+ -+ ret = ntsync_schedule(q, &args); -+ -+ /* and finally, unqueue */ -+ -+ for (i = 0; i < total_count; i++) { -+ struct ntsync_q_entry *entry = &q->entries[i]; -+ struct ntsync_obj *obj = entry->obj; -+ -+ all = ntsync_lock_obj(dev, obj); -+ list_del(&entry->node); -+ ntsync_unlock_obj(dev, obj, all); -+ -+ put_obj(obj); -+ } -+ -+ signaled = atomic_read(&q->signaled); -+ if (signaled != -1) { -+ struct ntsync_wait_args __user *user_args = argp; -+ -+ /* even if we caught a signal, we need to communicate success */ -+ ret = q->ownerdead ? -EOWNERDEAD : 0; -+ -+ if (put_user(signaled, &user_args->index)) -+ ret = -EFAULT; -+ } else if (!ret) { -+ ret = -ETIMEDOUT; -+ } -+ -+ kfree(q); -+ return ret; -+} -+ -+static int ntsync_wait_all(struct ntsync_device *dev, void __user *argp) -+{ -+ struct ntsync_wait_args args; -+ struct ntsync_q *q; -+ int signaled; -+ __u32 i; -+ int ret; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ ret = setup_wait(dev, &args, true, &q); -+ if (ret < 0) -+ return ret; -+ -+ /* queue ourselves */ -+ -+ mutex_lock(&dev->wait_all_lock); -+ -+ for (i = 0; i < args.count; i++) { -+ struct ntsync_q_entry *entry = &q->entries[i]; -+ struct ntsync_obj *obj = entry->obj; -+ -+ atomic_inc(&obj->all_hint); -+ -+ /* -+ * obj->all_waiters is protected by dev->wait_all_lock rather -+ * than obj->lock, so there is no need to acquire obj->lock -+ * here. -+ */ -+ list_add_tail(&entry->node, &obj->all_waiters); -+ } -+ if (args.alert) { -+ struct ntsync_q_entry *entry = &q->entries[args.count]; -+ struct ntsync_obj *obj = entry->obj; -+ -+ dev_lock_obj(dev, obj); -+ list_add_tail(&entry->node, &obj->any_waiters); -+ dev_unlock_obj(dev, obj); -+ } -+ -+ /* check if we are already signaled */ -+ -+ try_wake_all(dev, q, NULL); -+ -+ mutex_unlock(&dev->wait_all_lock); -+ -+ /* -+ * Check if the alert event is signaled, making sure to do so only -+ * after checking if the other objects are signaled. -+ */ -+ -+ if (args.alert) { -+ struct ntsync_obj *obj = q->entries[args.count].obj; -+ -+ if (atomic_read(&q->signaled) == -1) { -+ bool all = ntsync_lock_obj(dev, obj); -+ try_wake_any_obj(obj); -+ ntsync_unlock_obj(dev, obj, all); -+ } -+ } -+ -+ /* sleep */ -+ -+ ret = ntsync_schedule(q, &args); -+ -+ /* and finally, unqueue */ -+ -+ mutex_lock(&dev->wait_all_lock); -+ -+ for (i = 0; i < args.count; i++) { -+ struct ntsync_q_entry *entry = &q->entries[i]; -+ struct ntsync_obj *obj = entry->obj; -+ -+ /* -+ * obj->all_waiters is protected by dev->wait_all_lock rather -+ * than obj->lock, so there is no need to acquire it here. -+ */ -+ list_del(&entry->node); -+ -+ atomic_dec(&obj->all_hint); -+ -+ put_obj(obj); -+ } -+ -+ mutex_unlock(&dev->wait_all_lock); -+ -+ if (args.alert) { -+ struct ntsync_q_entry *entry = &q->entries[args.count]; -+ struct ntsync_obj *obj = entry->obj; -+ bool all; -+ -+ all = ntsync_lock_obj(dev, obj); -+ list_del(&entry->node); -+ ntsync_unlock_obj(dev, obj, all); -+ -+ put_obj(obj); -+ } -+ -+ signaled = atomic_read(&q->signaled); -+ if (signaled != -1) { -+ struct ntsync_wait_args __user *user_args = argp; -+ -+ /* even if we caught a signal, we need to communicate success */ -+ ret = q->ownerdead ? -EOWNERDEAD : 0; -+ -+ if (put_user(signaled, &user_args->index)) -+ ret = -EFAULT; -+ } else if (!ret) { -+ ret = -ETIMEDOUT; -+ } -+ -+ kfree(q); -+ return ret; - } - - static int ntsync_char_open(struct inode *inode, struct file *file) -@@ -198,6 +1158,8 @@ static int ntsync_char_open(struct inode *inode, struct file *file) - if (!dev) - return -ENOMEM; - -+ mutex_init(&dev->wait_all_lock); -+ - file->private_data = dev; - dev->file = file; - return nonseekable_open(inode, file); -@@ -219,8 +1181,16 @@ static long ntsync_char_ioctl(struct file *file, unsigned int cmd, - void __user *argp = (void __user *)parm; - - switch (cmd) { -+ case NTSYNC_IOC_CREATE_EVENT: -+ return ntsync_create_event(dev, argp); -+ case NTSYNC_IOC_CREATE_MUTEX: -+ return ntsync_create_mutex(dev, argp); - case NTSYNC_IOC_CREATE_SEM: - return ntsync_create_sem(dev, argp); -+ case NTSYNC_IOC_WAIT_ALL: -+ return ntsync_wait_all(dev, argp); -+ case NTSYNC_IOC_WAIT_ANY: -+ return ntsync_wait_any(dev, argp); - default: - return -ENOIOCTLCMD; - } -diff --git a/include/uapi/linux/ntsync.h b/include/uapi/linux/ntsync.h -index dcfa38fdc93c..6d06793512b1 100644 ---- a/include/uapi/linux/ntsync.h -+++ b/include/uapi/linux/ntsync.h -@@ -11,13 +11,49 @@ - #include - - struct ntsync_sem_args { -- __u32 sem; - __u32 count; - __u32 max; - }; - --#define NTSYNC_IOC_CREATE_SEM _IOWR('N', 0x80, struct ntsync_sem_args) -+struct ntsync_mutex_args { -+ __u32 owner; -+ __u32 count; -+}; -+ -+struct ntsync_event_args { -+ __u32 manual; -+ __u32 signaled; -+}; -+ -+#define NTSYNC_WAIT_REALTIME 0x1 -+ -+struct ntsync_wait_args { -+ __u64 timeout; -+ __u64 objs; -+ __u32 count; -+ __u32 index; -+ __u32 flags; -+ __u32 owner; -+ __u32 alert; -+ __u32 pad; -+}; -+ -+#define NTSYNC_MAX_WAIT_COUNT 64 -+ -+#define NTSYNC_IOC_CREATE_SEM _IOW ('N', 0x80, struct ntsync_sem_args) -+#define NTSYNC_IOC_WAIT_ANY _IOWR('N', 0x82, struct ntsync_wait_args) -+#define NTSYNC_IOC_WAIT_ALL _IOWR('N', 0x83, struct ntsync_wait_args) -+#define NTSYNC_IOC_CREATE_MUTEX _IOW ('N', 0x84, struct ntsync_mutex_args) -+#define NTSYNC_IOC_CREATE_EVENT _IOW ('N', 0x87, struct ntsync_event_args) - --#define NTSYNC_IOC_SEM_POST _IOWR('N', 0x81, __u32) -+#define NTSYNC_IOC_SEM_RELEASE _IOWR('N', 0x81, __u32) -+#define NTSYNC_IOC_MUTEX_UNLOCK _IOWR('N', 0x85, struct ntsync_mutex_args) -+#define NTSYNC_IOC_MUTEX_KILL _IOW ('N', 0x86, __u32) -+#define NTSYNC_IOC_EVENT_SET _IOR ('N', 0x88, __u32) -+#define NTSYNC_IOC_EVENT_RESET _IOR ('N', 0x89, __u32) -+#define NTSYNC_IOC_EVENT_PULSE _IOR ('N', 0x8a, __u32) -+#define NTSYNC_IOC_SEM_READ _IOR ('N', 0x8b, struct ntsync_sem_args) -+#define NTSYNC_IOC_MUTEX_READ _IOR ('N', 0x8c, struct ntsync_mutex_args) -+#define NTSYNC_IOC_EVENT_READ _IOR ('N', 0x8d, struct ntsync_event_args) - - #endif -diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile -index 2401e973c359..a8c9648e5adc 100644 ---- a/tools/testing/selftests/Makefile -+++ b/tools/testing/selftests/Makefile -@@ -18,6 +18,7 @@ TARGETS += devices/error_logs - TARGETS += devices/probe - TARGETS += dmabuf-heaps - TARGETS += drivers/dma-buf -+TARGETS += drivers/ntsync - TARGETS += drivers/s390x/uvdevice - TARGETS += drivers/net - TARGETS += drivers/net/bonding -diff --git a/tools/testing/selftests/drivers/ntsync/.gitignore b/tools/testing/selftests/drivers/ntsync/.gitignore -new file mode 100644 -index 000000000000..848573a3d3ea ---- /dev/null -+++ b/tools/testing/selftests/drivers/ntsync/.gitignore -@@ -0,0 +1 @@ -+ntsync -diff --git a/tools/testing/selftests/drivers/ntsync/Makefile b/tools/testing/selftests/drivers/ntsync/Makefile -new file mode 100644 -index 000000000000..dbf2b055c0b2 ---- /dev/null -+++ b/tools/testing/selftests/drivers/ntsync/Makefile -@@ -0,0 +1,7 @@ -+# SPDX-LICENSE-IDENTIFIER: GPL-2.0-only -+TEST_GEN_PROGS := ntsync -+ -+CFLAGS += $(KHDR_INCLUDES) -+LDLIBS += -lpthread -+ -+include ../../lib.mk -diff --git a/tools/testing/selftests/drivers/ntsync/config b/tools/testing/selftests/drivers/ntsync/config -new file mode 100644 -index 000000000000..60539c826d06 ---- /dev/null -+++ b/tools/testing/selftests/drivers/ntsync/config -@@ -0,0 +1 @@ -+CONFIG_WINESYNC=y -diff --git a/tools/testing/selftests/drivers/ntsync/ntsync.c b/tools/testing/selftests/drivers/ntsync/ntsync.c -new file mode 100644 -index 000000000000..3aad311574c4 ---- /dev/null -+++ b/tools/testing/selftests/drivers/ntsync/ntsync.c -@@ -0,0 +1,1343 @@ -+// SPDX-License-Identifier: GPL-2.0-or-later -+/* -+ * Various unit tests for the "ntsync" synchronization primitive driver. -+ * -+ * Copyright (C) 2021-2022 Elizabeth Figura -+ */ -+ -+#define _GNU_SOURCE -+#include -+#include -+#include -+#include -+#include -+#include -+#include "../../kselftest_harness.h" -+ -+static int read_sem_state(int sem, __u32 *count, __u32 *max) -+{ -+ struct ntsync_sem_args args; -+ int ret; -+ -+ memset(&args, 0xcc, sizeof(args)); -+ ret = ioctl(sem, NTSYNC_IOC_SEM_READ, &args); -+ *count = args.count; -+ *max = args.max; -+ return ret; -+} -+ -+#define check_sem_state(sem, count, max) \ -+ ({ \ -+ __u32 __count, __max; \ -+ int ret = read_sem_state((sem), &__count, &__max); \ -+ EXPECT_EQ(0, ret); \ -+ EXPECT_EQ((count), __count); \ -+ EXPECT_EQ((max), __max); \ -+ }) -+ -+static int release_sem(int sem, __u32 *count) -+{ -+ return ioctl(sem, NTSYNC_IOC_SEM_RELEASE, count); -+} -+ -+static int read_mutex_state(int mutex, __u32 *count, __u32 *owner) -+{ -+ struct ntsync_mutex_args args; -+ int ret; -+ -+ memset(&args, 0xcc, sizeof(args)); -+ ret = ioctl(mutex, NTSYNC_IOC_MUTEX_READ, &args); -+ *count = args.count; -+ *owner = args.owner; -+ return ret; -+} -+ -+#define check_mutex_state(mutex, count, owner) \ -+ ({ \ -+ __u32 __count, __owner; \ -+ int ret = read_mutex_state((mutex), &__count, &__owner); \ -+ EXPECT_EQ(0, ret); \ -+ EXPECT_EQ((count), __count); \ -+ EXPECT_EQ((owner), __owner); \ -+ }) -+ -+static int unlock_mutex(int mutex, __u32 owner, __u32 *count) -+{ -+ struct ntsync_mutex_args args; -+ int ret; -+ -+ args.owner = owner; -+ args.count = 0xdeadbeef; -+ ret = ioctl(mutex, NTSYNC_IOC_MUTEX_UNLOCK, &args); -+ *count = args.count; -+ return ret; -+} -+ -+static int read_event_state(int event, __u32 *signaled, __u32 *manual) -+{ -+ struct ntsync_event_args args; -+ int ret; -+ -+ memset(&args, 0xcc, sizeof(args)); -+ ret = ioctl(event, NTSYNC_IOC_EVENT_READ, &args); -+ *signaled = args.signaled; -+ *manual = args.manual; -+ return ret; -+} -+ -+#define check_event_state(event, signaled, manual) \ -+ ({ \ -+ __u32 __signaled, __manual; \ -+ int ret = read_event_state((event), &__signaled, &__manual); \ -+ EXPECT_EQ(0, ret); \ -+ EXPECT_EQ((signaled), __signaled); \ -+ EXPECT_EQ((manual), __manual); \ -+ }) -+ -+static int wait_objs(int fd, unsigned long request, __u32 count, -+ const int *objs, __u32 owner, int alert, __u32 *index) -+{ -+ struct ntsync_wait_args args = {0}; -+ struct timespec timeout; -+ int ret; -+ -+ clock_gettime(CLOCK_MONOTONIC, &timeout); -+ -+ args.timeout = timeout.tv_sec * 1000000000 + timeout.tv_nsec; -+ args.count = count; -+ args.objs = (uintptr_t)objs; -+ args.owner = owner; -+ args.index = 0xdeadbeef; -+ args.alert = alert; -+ ret = ioctl(fd, request, &args); -+ *index = args.index; -+ return ret; -+} -+ -+static int wait_any(int fd, __u32 count, const int *objs, __u32 owner, __u32 *index) -+{ -+ return wait_objs(fd, NTSYNC_IOC_WAIT_ANY, count, objs, owner, 0, index); -+} -+ -+static int wait_all(int fd, __u32 count, const int *objs, __u32 owner, __u32 *index) -+{ -+ return wait_objs(fd, NTSYNC_IOC_WAIT_ALL, count, objs, owner, 0, index); -+} -+ -+static int wait_any_alert(int fd, __u32 count, const int *objs, -+ __u32 owner, int alert, __u32 *index) -+{ -+ return wait_objs(fd, NTSYNC_IOC_WAIT_ANY, -+ count, objs, owner, alert, index); -+} -+ -+static int wait_all_alert(int fd, __u32 count, const int *objs, -+ __u32 owner, int alert, __u32 *index) -+{ -+ return wait_objs(fd, NTSYNC_IOC_WAIT_ALL, -+ count, objs, owner, alert, index); -+} -+ -+TEST(semaphore_state) -+{ -+ struct ntsync_sem_args sem_args; -+ struct timespec timeout; -+ __u32 count, index; -+ int fd, ret, sem; -+ -+ clock_gettime(CLOCK_MONOTONIC, &timeout); -+ -+ fd = open("/dev/ntsync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 3; -+ sem_args.max = 2; -+ sem = ioctl(fd, NTSYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(-1, sem); -+ EXPECT_EQ(EINVAL, errno); -+ -+ sem_args.count = 2; -+ sem_args.max = 2; -+ sem = ioctl(fd, NTSYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_LE(0, sem); -+ check_sem_state(sem, 2, 2); -+ -+ count = 0; -+ ret = release_sem(sem, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, count); -+ check_sem_state(sem, 2, 2); -+ -+ count = 1; -+ ret = release_sem(sem, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOVERFLOW, errno); -+ check_sem_state(sem, 2, 2); -+ -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(sem, 1, 2); -+ -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(sem, 0, 2); -+ -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ count = 3; -+ ret = release_sem(sem, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOVERFLOW, errno); -+ check_sem_state(sem, 0, 2); -+ -+ count = 2; -+ ret = release_sem(sem, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, count); -+ check_sem_state(sem, 2, 2); -+ -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(0, ret); -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(0, ret); -+ -+ count = 1; -+ ret = release_sem(sem, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, count); -+ check_sem_state(sem, 1, 2); -+ -+ count = ~0u; -+ ret = release_sem(sem, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOVERFLOW, errno); -+ check_sem_state(sem, 1, 2); -+ -+ close(sem); -+ -+ close(fd); -+} -+ -+TEST(mutex_state) -+{ -+ struct ntsync_mutex_args mutex_args; -+ __u32 owner, count, index; -+ struct timespec timeout; -+ int fd, ret, mutex; -+ -+ clock_gettime(CLOCK_MONOTONIC, &timeout); -+ -+ fd = open("/dev/ntsync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ mutex_args.owner = 123; -+ mutex_args.count = 0; -+ mutex = ioctl(fd, NTSYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, mutex); -+ EXPECT_EQ(EINVAL, errno); -+ -+ mutex_args.owner = 0; -+ mutex_args.count = 2; -+ mutex = ioctl(fd, NTSYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, mutex); -+ EXPECT_EQ(EINVAL, errno); -+ -+ mutex_args.owner = 123; -+ mutex_args.count = 2; -+ mutex = ioctl(fd, NTSYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_LE(0, mutex); -+ check_mutex_state(mutex, 2, 123); -+ -+ ret = unlock_mutex(mutex, 0, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = unlock_mutex(mutex, 456, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EPERM, errno); -+ check_mutex_state(mutex, 2, 123); -+ -+ ret = unlock_mutex(mutex, 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, count); -+ check_mutex_state(mutex, 1, 123); -+ -+ ret = unlock_mutex(mutex, 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, count); -+ check_mutex_state(mutex, 0, 0); -+ -+ ret = unlock_mutex(mutex, 123, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EPERM, errno); -+ -+ ret = wait_any(fd, 1, &mutex, 456, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_mutex_state(mutex, 1, 456); -+ -+ ret = wait_any(fd, 1, &mutex, 456, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_mutex_state(mutex, 2, 456); -+ -+ ret = unlock_mutex(mutex, 456, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, count); -+ check_mutex_state(mutex, 1, 456); -+ -+ ret = wait_any(fd, 1, &mutex, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ owner = 0; -+ ret = ioctl(mutex, NTSYNC_IOC_MUTEX_KILL, &owner); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ owner = 123; -+ ret = ioctl(mutex, NTSYNC_IOC_MUTEX_KILL, &owner); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EPERM, errno); -+ check_mutex_state(mutex, 1, 456); -+ -+ owner = 456; -+ ret = ioctl(mutex, NTSYNC_IOC_MUTEX_KILL, &owner); -+ EXPECT_EQ(0, ret); -+ -+ memset(&mutex_args, 0xcc, sizeof(mutex_args)); -+ ret = ioctl(mutex, NTSYNC_IOC_MUTEX_READ, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, mutex_args.count); -+ EXPECT_EQ(0, mutex_args.owner); -+ -+ memset(&mutex_args, 0xcc, sizeof(mutex_args)); -+ ret = ioctl(mutex, NTSYNC_IOC_MUTEX_READ, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, mutex_args.count); -+ EXPECT_EQ(0, mutex_args.owner); -+ -+ ret = wait_any(fd, 1, &mutex, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, index); -+ check_mutex_state(mutex, 1, 123); -+ -+ owner = 123; -+ ret = ioctl(mutex, NTSYNC_IOC_MUTEX_KILL, &owner); -+ EXPECT_EQ(0, ret); -+ -+ memset(&mutex_args, 0xcc, sizeof(mutex_args)); -+ ret = ioctl(mutex, NTSYNC_IOC_MUTEX_READ, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, mutex_args.count); -+ EXPECT_EQ(0, mutex_args.owner); -+ -+ ret = wait_any(fd, 1, &mutex, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, index); -+ check_mutex_state(mutex, 1, 123); -+ -+ close(mutex); -+ -+ mutex_args.owner = 0; -+ mutex_args.count = 0; -+ mutex = ioctl(fd, NTSYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_LE(0, mutex); -+ check_mutex_state(mutex, 0, 0); -+ -+ ret = wait_any(fd, 1, &mutex, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_mutex_state(mutex, 1, 123); -+ -+ close(mutex); -+ -+ mutex_args.owner = 123; -+ mutex_args.count = ~0u; -+ mutex = ioctl(fd, NTSYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_LE(0, mutex); -+ check_mutex_state(mutex, ~0u, 123); -+ -+ ret = wait_any(fd, 1, &mutex, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ close(mutex); -+ -+ close(fd); -+} -+ -+TEST(manual_event_state) -+{ -+ struct ntsync_event_args event_args; -+ __u32 index, signaled; -+ int fd, event, ret; -+ -+ fd = open("/dev/ntsync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ event_args.manual = 1; -+ event_args.signaled = 0; -+ event = ioctl(fd, NTSYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_LE(0, event); -+ check_event_state(event, 0, 1); -+ -+ signaled = 0xdeadbeef; -+ ret = ioctl(event, NTSYNC_IOC_EVENT_SET, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, signaled); -+ check_event_state(event, 1, 1); -+ -+ ret = ioctl(event, NTSYNC_IOC_EVENT_SET, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, signaled); -+ check_event_state(event, 1, 1); -+ -+ ret = wait_any(fd, 1, &event, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_event_state(event, 1, 1); -+ -+ signaled = 0xdeadbeef; -+ ret = ioctl(event, NTSYNC_IOC_EVENT_RESET, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, signaled); -+ check_event_state(event, 0, 1); -+ -+ ret = ioctl(event, NTSYNC_IOC_EVENT_RESET, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, signaled); -+ check_event_state(event, 0, 1); -+ -+ ret = wait_any(fd, 1, &event, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ ret = ioctl(event, NTSYNC_IOC_EVENT_SET, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, signaled); -+ -+ ret = ioctl(event, NTSYNC_IOC_EVENT_PULSE, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, signaled); -+ check_event_state(event, 0, 1); -+ -+ ret = ioctl(event, NTSYNC_IOC_EVENT_PULSE, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, signaled); -+ check_event_state(event, 0, 1); -+ -+ close(event); -+ -+ close(fd); -+} -+ -+TEST(auto_event_state) -+{ -+ struct ntsync_event_args event_args; -+ __u32 index, signaled; -+ int fd, event, ret; -+ -+ fd = open("/dev/ntsync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ event_args.manual = 0; -+ event_args.signaled = 1; -+ event = ioctl(fd, NTSYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_LE(0, event); -+ -+ check_event_state(event, 1, 0); -+ -+ signaled = 0xdeadbeef; -+ ret = ioctl(event, NTSYNC_IOC_EVENT_SET, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, signaled); -+ check_event_state(event, 1, 0); -+ -+ ret = wait_any(fd, 1, &event, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_event_state(event, 0, 0); -+ -+ signaled = 0xdeadbeef; -+ ret = ioctl(event, NTSYNC_IOC_EVENT_RESET, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, signaled); -+ check_event_state(event, 0, 0); -+ -+ ret = wait_any(fd, 1, &event, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ ret = ioctl(event, NTSYNC_IOC_EVENT_SET, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, signaled); -+ -+ ret = ioctl(event, NTSYNC_IOC_EVENT_PULSE, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, signaled); -+ check_event_state(event, 0, 0); -+ -+ ret = ioctl(event, NTSYNC_IOC_EVENT_PULSE, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, signaled); -+ check_event_state(event, 0, 0); -+ -+ close(event); -+ -+ close(fd); -+} -+ -+TEST(test_wait_any) -+{ -+ int objs[NTSYNC_MAX_WAIT_COUNT + 1], fd, ret; -+ struct ntsync_mutex_args mutex_args = {0}; -+ struct ntsync_sem_args sem_args = {0}; -+ __u32 owner, index, count, i; -+ struct timespec timeout; -+ -+ clock_gettime(CLOCK_MONOTONIC, &timeout); -+ -+ fd = open("/dev/ntsync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 2; -+ sem_args.max = 3; -+ objs[0] = ioctl(fd, NTSYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_LE(0, objs[0]); -+ -+ mutex_args.owner = 0; -+ mutex_args.count = 0; -+ objs[1] = ioctl(fd, NTSYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_LE(0, objs[1]); -+ -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(objs[0], 1, 3); -+ check_mutex_state(objs[1], 0, 0); -+ -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(objs[0], 0, 3); -+ check_mutex_state(objs[1], 0, 0); -+ -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, index); -+ check_sem_state(objs[0], 0, 3); -+ check_mutex_state(objs[1], 1, 123); -+ -+ count = 1; -+ ret = release_sem(objs[0], &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, count); -+ -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(objs[0], 0, 3); -+ check_mutex_state(objs[1], 1, 123); -+ -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, index); -+ check_sem_state(objs[0], 0, 3); -+ check_mutex_state(objs[1], 2, 123); -+ -+ ret = wait_any(fd, 2, objs, 456, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ owner = 123; -+ ret = ioctl(objs[1], NTSYNC_IOC_MUTEX_KILL, &owner); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_any(fd, 2, objs, 456, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(1, index); -+ -+ ret = wait_any(fd, 2, objs, 456, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, index); -+ -+ close(objs[1]); -+ -+ /* test waiting on the same object twice */ -+ -+ count = 2; -+ ret = release_sem(objs[0], &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, count); -+ -+ objs[1] = objs[0]; -+ ret = wait_any(fd, 2, objs, 456, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(objs[0], 1, 3); -+ -+ ret = wait_any(fd, 0, NULL, 456, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ for (i = 1; i < NTSYNC_MAX_WAIT_COUNT + 1; ++i) -+ objs[i] = objs[0]; -+ -+ ret = wait_any(fd, NTSYNC_MAX_WAIT_COUNT, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ ret = wait_any(fd, NTSYNC_MAX_WAIT_COUNT + 1, objs, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = wait_any(fd, -1, objs, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ close(objs[0]); -+ -+ close(fd); -+} -+ -+TEST(test_wait_all) -+{ -+ struct ntsync_event_args event_args = {0}; -+ struct ntsync_mutex_args mutex_args = {0}; -+ struct ntsync_sem_args sem_args = {0}; -+ __u32 owner, index, count; -+ int objs[2], fd, ret; -+ -+ fd = open("/dev/ntsync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 2; -+ sem_args.max = 3; -+ objs[0] = ioctl(fd, NTSYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_LE(0, objs[0]); -+ -+ mutex_args.owner = 0; -+ mutex_args.count = 0; -+ objs[1] = ioctl(fd, NTSYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_LE(0, objs[1]); -+ -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(objs[0], 1, 3); -+ check_mutex_state(objs[1], 1, 123); -+ -+ ret = wait_all(fd, 2, objs, 456, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ check_sem_state(objs[0], 1, 3); -+ check_mutex_state(objs[1], 1, 123); -+ -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(objs[0], 0, 3); -+ check_mutex_state(objs[1], 2, 123); -+ -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ check_sem_state(objs[0], 0, 3); -+ check_mutex_state(objs[1], 2, 123); -+ -+ count = 3; -+ ret = release_sem(objs[0], &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, count); -+ -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(objs[0], 2, 3); -+ check_mutex_state(objs[1], 3, 123); -+ -+ owner = 123; -+ ret = ioctl(objs[1], NTSYNC_IOC_MUTEX_KILL, &owner); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ check_sem_state(objs[0], 1, 3); -+ check_mutex_state(objs[1], 1, 123); -+ -+ close(objs[1]); -+ -+ event_args.manual = true; -+ event_args.signaled = true; -+ objs[1] = ioctl(fd, NTSYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_LE(0, objs[1]); -+ -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(objs[0], 0, 3); -+ check_event_state(objs[1], 1, 1); -+ -+ close(objs[1]); -+ -+ /* test waiting on the same object twice */ -+ objs[1] = objs[0]; -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ close(objs[0]); -+ -+ close(fd); -+} -+ -+struct wake_args { -+ int fd; -+ int obj; -+}; -+ -+struct wait_args { -+ int fd; -+ unsigned long request; -+ struct ntsync_wait_args *args; -+ int ret; -+ int err; -+}; -+ -+static void *wait_thread(void *arg) -+{ -+ struct wait_args *args = arg; -+ -+ args->ret = ioctl(args->fd, args->request, args->args); -+ args->err = errno; -+ return NULL; -+} -+ -+static __u64 get_abs_timeout(unsigned int ms) -+{ -+ struct timespec timeout; -+ clock_gettime(CLOCK_MONOTONIC, &timeout); -+ return (timeout.tv_sec * 1000000000) + timeout.tv_nsec + (ms * 1000000); -+} -+ -+static int wait_for_thread(pthread_t thread, unsigned int ms) -+{ -+ struct timespec timeout; -+ -+ clock_gettime(CLOCK_REALTIME, &timeout); -+ timeout.tv_nsec += ms * 1000000; -+ timeout.tv_sec += (timeout.tv_nsec / 1000000000); -+ timeout.tv_nsec %= 1000000000; -+ return pthread_timedjoin_np(thread, NULL, &timeout); -+} -+ -+TEST(wake_any) -+{ -+ struct ntsync_event_args event_args = {0}; -+ struct ntsync_mutex_args mutex_args = {0}; -+ struct ntsync_wait_args wait_args = {0}; -+ struct ntsync_sem_args sem_args = {0}; -+ struct wait_args thread_args; -+ __u32 count, index, signaled; -+ int objs[2], fd, ret; -+ pthread_t thread; -+ -+ fd = open("/dev/ntsync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 0; -+ sem_args.max = 3; -+ objs[0] = ioctl(fd, NTSYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_LE(0, objs[0]); -+ -+ mutex_args.owner = 123; -+ mutex_args.count = 1; -+ objs[1] = ioctl(fd, NTSYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_LE(0, objs[1]); -+ -+ /* test waking the semaphore */ -+ -+ wait_args.timeout = get_abs_timeout(1000); -+ wait_args.objs = (uintptr_t)objs; -+ wait_args.count = 2; -+ wait_args.owner = 456; -+ wait_args.index = 0xdeadbeef; -+ thread_args.fd = fd; -+ thread_args.args = &wait_args; -+ thread_args.request = NTSYNC_IOC_WAIT_ANY; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ count = 1; -+ ret = release_sem(objs[0], &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, count); -+ check_sem_state(objs[0], 0, 3); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(0, wait_args.index); -+ -+ /* test waking the mutex */ -+ -+ /* first grab it again for owner 123 */ -+ ret = wait_any(fd, 1, &objs[1], 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ wait_args.timeout = get_abs_timeout(1000); -+ wait_args.owner = 456; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = unlock_mutex(objs[1], 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, count); -+ -+ ret = pthread_tryjoin_np(thread, NULL); -+ EXPECT_EQ(EBUSY, ret); -+ -+ ret = unlock_mutex(objs[1], 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, mutex_args.count); -+ check_mutex_state(objs[1], 1, 456); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); -+ -+ close(objs[1]); -+ -+ /* test waking events */ -+ -+ event_args.manual = false; -+ event_args.signaled = false; -+ objs[1] = ioctl(fd, NTSYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_LE(0, objs[1]); -+ -+ wait_args.timeout = get_abs_timeout(1000); -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(objs[1], NTSYNC_IOC_EVENT_SET, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, signaled); -+ check_event_state(objs[1], 0, 0); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); -+ -+ wait_args.timeout = get_abs_timeout(1000); -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(objs[1], NTSYNC_IOC_EVENT_PULSE, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, signaled); -+ check_event_state(objs[1], 0, 0); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); -+ -+ close(objs[1]); -+ -+ event_args.manual = true; -+ event_args.signaled = false; -+ objs[1] = ioctl(fd, NTSYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_LE(0, objs[1]); -+ -+ wait_args.timeout = get_abs_timeout(1000); -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(objs[1], NTSYNC_IOC_EVENT_SET, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, signaled); -+ check_event_state(objs[1], 1, 1); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); -+ -+ ret = ioctl(objs[1], NTSYNC_IOC_EVENT_RESET, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, signaled); -+ -+ wait_args.timeout = get_abs_timeout(1000); -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(objs[1], NTSYNC_IOC_EVENT_PULSE, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, signaled); -+ check_event_state(objs[1], 0, 1); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); -+ -+ /* delete an object while it's being waited on */ -+ -+ wait_args.timeout = get_abs_timeout(200); -+ wait_args.owner = 123; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ close(objs[0]); -+ close(objs[1]); -+ -+ ret = wait_for_thread(thread, 200); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(-1, thread_args.ret); -+ EXPECT_EQ(ETIMEDOUT, thread_args.err); -+ -+ close(fd); -+} -+ -+TEST(wake_all) -+{ -+ struct ntsync_event_args manual_event_args = {0}; -+ struct ntsync_event_args auto_event_args = {0}; -+ struct ntsync_mutex_args mutex_args = {0}; -+ struct ntsync_wait_args wait_args = {0}; -+ struct ntsync_sem_args sem_args = {0}; -+ struct wait_args thread_args; -+ __u32 count, index, signaled; -+ int objs[4], fd, ret; -+ pthread_t thread; -+ -+ fd = open("/dev/ntsync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 0; -+ sem_args.max = 3; -+ objs[0] = ioctl(fd, NTSYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_LE(0, objs[0]); -+ -+ mutex_args.owner = 123; -+ mutex_args.count = 1; -+ objs[1] = ioctl(fd, NTSYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_LE(0, objs[1]); -+ -+ manual_event_args.manual = true; -+ manual_event_args.signaled = true; -+ objs[2] = ioctl(fd, NTSYNC_IOC_CREATE_EVENT, &manual_event_args); -+ EXPECT_LE(0, objs[2]); -+ -+ auto_event_args.manual = false; -+ auto_event_args.signaled = true; -+ objs[3] = ioctl(fd, NTSYNC_IOC_CREATE_EVENT, &auto_event_args); -+ EXPECT_EQ(0, objs[3]); -+ -+ wait_args.timeout = get_abs_timeout(1000); -+ wait_args.objs = (uintptr_t)objs; -+ wait_args.count = 4; -+ wait_args.owner = 456; -+ thread_args.fd = fd; -+ thread_args.args = &wait_args; -+ thread_args.request = NTSYNC_IOC_WAIT_ALL; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ count = 1; -+ ret = release_sem(objs[0], &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, count); -+ -+ ret = pthread_tryjoin_np(thread, NULL); -+ EXPECT_EQ(EBUSY, ret); -+ -+ check_sem_state(objs[0], 1, 3); -+ -+ ret = wait_any(fd, 1, &objs[0], 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ ret = unlock_mutex(objs[1], 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, count); -+ -+ ret = pthread_tryjoin_np(thread, NULL); -+ EXPECT_EQ(EBUSY, ret); -+ -+ check_mutex_state(objs[1], 0, 0); -+ -+ ret = ioctl(objs[2], NTSYNC_IOC_EVENT_RESET, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, signaled); -+ -+ count = 2; -+ ret = release_sem(objs[0], &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, count); -+ check_sem_state(objs[0], 2, 3); -+ -+ ret = ioctl(objs[3], NTSYNC_IOC_EVENT_RESET, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, signaled); -+ -+ ret = ioctl(objs[2], NTSYNC_IOC_EVENT_SET, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, signaled); -+ -+ ret = ioctl(objs[3], NTSYNC_IOC_EVENT_SET, &signaled); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, signaled); -+ -+ check_sem_state(objs[0], 1, 3); -+ check_mutex_state(objs[1], 1, 456); -+ check_event_state(objs[2], 1, 1); -+ check_event_state(objs[3], 0, 0); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ -+ /* delete an object while it's being waited on */ -+ -+ wait_args.timeout = get_abs_timeout(200); -+ wait_args.owner = 123; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ close(objs[0]); -+ close(objs[1]); -+ close(objs[2]); -+ close(objs[3]); -+ -+ ret = wait_for_thread(thread, 200); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(-1, thread_args.ret); -+ EXPECT_EQ(ETIMEDOUT, thread_args.err); -+ -+ close(fd); -+} -+ -+TEST(alert_any) -+{ -+ struct ntsync_event_args event_args = {0}; -+ struct ntsync_wait_args wait_args = {0}; -+ struct ntsync_sem_args sem_args = {0}; -+ __u32 index, count, signaled; -+ struct wait_args thread_args; -+ int objs[2], event, fd, ret; -+ pthread_t thread; -+ -+ fd = open("/dev/ntsync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 0; -+ sem_args.max = 2; -+ objs[0] = ioctl(fd, NTSYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_LE(0, objs[0]); -+ -+ sem_args.count = 1; -+ sem_args.max = 2; -+ objs[1] = ioctl(fd, NTSYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_LE(0, objs[1]); -+ -+ event_args.manual = true; -+ event_args.signaled = true; -+ event = ioctl(fd, NTSYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_LE(0, event); -+ -+ ret = wait_any_alert(fd, 0, NULL, 123, event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ ret = ioctl(event, NTSYNC_IOC_EVENT_RESET, &signaled); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_any_alert(fd, 0, NULL, 123, event, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ ret = ioctl(event, NTSYNC_IOC_EVENT_SET, &signaled); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_any_alert(fd, 2, objs, 123, event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, index); -+ -+ ret = wait_any_alert(fd, 2, objs, 123, event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, index); -+ -+ /* test wakeup via alert */ -+ -+ ret = ioctl(event, NTSYNC_IOC_EVENT_RESET, &signaled); -+ EXPECT_EQ(0, ret); -+ -+ wait_args.timeout = get_abs_timeout(1000); -+ wait_args.objs = (uintptr_t)objs; -+ wait_args.count = 2; -+ wait_args.owner = 123; -+ wait_args.index = 0xdeadbeef; -+ wait_args.alert = event; -+ thread_args.fd = fd; -+ thread_args.args = &wait_args; -+ thread_args.request = NTSYNC_IOC_WAIT_ANY; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(event, NTSYNC_IOC_EVENT_SET, &signaled); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(2, wait_args.index); -+ -+ close(event); -+ -+ /* test with an auto-reset event */ -+ -+ event_args.manual = false; -+ event_args.signaled = true; -+ event = ioctl(fd, NTSYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_LE(0, event); -+ -+ count = 1; -+ ret = release_sem(objs[0], &count); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_any_alert(fd, 2, objs, 123, event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ ret = wait_any_alert(fd, 2, objs, 123, event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, index); -+ -+ ret = wait_any_alert(fd, 2, objs, 123, event, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ close(event); -+ -+ close(objs[0]); -+ close(objs[1]); -+ -+ close(fd); -+} -+ -+TEST(alert_all) -+{ -+ struct ntsync_event_args event_args = {0}; -+ struct ntsync_wait_args wait_args = {0}; -+ struct ntsync_sem_args sem_args = {0}; -+ struct wait_args thread_args; -+ __u32 index, count, signaled; -+ int objs[2], event, fd, ret; -+ pthread_t thread; -+ -+ fd = open("/dev/ntsync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 2; -+ sem_args.max = 2; -+ objs[0] = ioctl(fd, NTSYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_LE(0, objs[0]); -+ -+ sem_args.count = 1; -+ sem_args.max = 2; -+ objs[1] = ioctl(fd, NTSYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_LE(0, objs[1]); -+ -+ event_args.manual = true; -+ event_args.signaled = true; -+ event = ioctl(fd, NTSYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_LE(0, event); -+ -+ ret = wait_all_alert(fd, 2, objs, 123, event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ ret = wait_all_alert(fd, 2, objs, 123, event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, index); -+ -+ /* test wakeup via alert */ -+ -+ ret = ioctl(event, NTSYNC_IOC_EVENT_RESET, &signaled); -+ EXPECT_EQ(0, ret); -+ -+ wait_args.timeout = get_abs_timeout(1000); -+ wait_args.objs = (uintptr_t)objs; -+ wait_args.count = 2; -+ wait_args.owner = 123; -+ wait_args.index = 0xdeadbeef; -+ wait_args.alert = event; -+ thread_args.fd = fd; -+ thread_args.args = &wait_args; -+ thread_args.request = NTSYNC_IOC_WAIT_ALL; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(event, NTSYNC_IOC_EVENT_SET, &signaled); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(2, wait_args.index); -+ -+ close(event); -+ -+ /* test with an auto-reset event */ -+ -+ event_args.manual = false; -+ event_args.signaled = true; -+ event = ioctl(fd, NTSYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_LE(0, event); -+ -+ count = 2; -+ ret = release_sem(objs[1], &count); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_all_alert(fd, 2, objs, 123, event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ ret = wait_all_alert(fd, 2, objs, 123, event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, index); -+ -+ ret = wait_all_alert(fd, 2, objs, 123, event, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ close(event); -+ -+ close(objs[0]); -+ close(objs[1]); -+ -+ close(fd); -+} -+ -+#define STRESS_LOOPS 10000 -+#define STRESS_THREADS 4 -+ -+static unsigned int stress_counter; -+static int stress_device, stress_start_event, stress_mutex; -+ -+static void *stress_thread(void *arg) -+{ -+ struct ntsync_wait_args wait_args = {0}; -+ __u32 index, count, i; -+ int ret; -+ -+ wait_args.timeout = UINT64_MAX; -+ wait_args.count = 1; -+ wait_args.objs = (uintptr_t)&stress_start_event; -+ wait_args.owner = gettid(); -+ wait_args.index = 0xdeadbeef; -+ -+ ioctl(stress_device, NTSYNC_IOC_WAIT_ANY, &wait_args); -+ -+ wait_args.objs = (uintptr_t)&stress_mutex; -+ -+ for (i = 0; i < STRESS_LOOPS; ++i) { -+ ioctl(stress_device, NTSYNC_IOC_WAIT_ANY, &wait_args); -+ -+ ++stress_counter; -+ -+ unlock_mutex(stress_mutex, wait_args.owner, &count); -+ } -+ -+ return NULL; -+} -+ -+TEST(stress_wait) -+{ -+ struct ntsync_event_args event_args; -+ struct ntsync_mutex_args mutex_args; -+ pthread_t threads[STRESS_THREADS]; -+ __u32 signaled, i; -+ int ret; -+ -+ stress_device = open("/dev/ntsync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, stress_device); -+ -+ mutex_args.owner = 0; -+ mutex_args.count = 0; -+ stress_mutex = ioctl(stress_device, NTSYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_LE(0, stress_mutex); -+ -+ event_args.manual = 1; -+ event_args.signaled = 0; -+ stress_start_event = ioctl(stress_device, NTSYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_LE(0, stress_start_event); -+ -+ for (i = 0; i < STRESS_THREADS; ++i) -+ pthread_create(&threads[i], NULL, stress_thread, NULL); -+ -+ ret = ioctl(stress_start_event, NTSYNC_IOC_EVENT_SET, &signaled); -+ EXPECT_EQ(0, ret); -+ -+ for (i = 0; i < STRESS_THREADS; ++i) { -+ ret = pthread_join(threads[i], NULL); -+ EXPECT_EQ(0, ret); -+ } -+ -+ EXPECT_EQ(STRESS_LOOPS * STRESS_THREADS, stress_counter); -+ -+ close(stress_start_event); -+ close(stress_mutex); -+ close(stress_device); -+} -+ -+TEST_HARNESS_MAIN --- -2.48.0.rc1 - -From 11d0eb9308e917e02ad88da6964e7e685d09b79a Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Sun, 2 Feb 2025 13:54:54 +0100 -Subject: [PATCH 09/12] perf-per-core - -Signed-off-by: Peter Jung ---- - Documentation/arch/x86/topology.rst | 4 + - arch/x86/events/rapl.c | 415 ++++++++++++++++---------- - arch/x86/include/asm/processor.h | 1 + - arch/x86/include/asm/topology.h | 1 + - arch/x86/kernel/cpu/debugfs.c | 1 + - arch/x86/kernel/cpu/topology_common.c | 1 + - 6 files changed, 273 insertions(+), 150 deletions(-) - -diff --git a/Documentation/arch/x86/topology.rst b/Documentation/arch/x86/topology.rst -index 7352ab89a55a..c12837e61bda 100644 ---- a/Documentation/arch/x86/topology.rst -+++ b/Documentation/arch/x86/topology.rst -@@ -135,6 +135,10 @@ Thread-related topology information in the kernel: - The ID of the core to which a thread belongs. It is also printed in /proc/cpuinfo - "core_id." - -+ - topology_logical_core_id(); -+ -+ The logical core ID to which a thread belongs. -+ - - - System topology examples -diff --git a/arch/x86/events/rapl.c b/arch/x86/events/rapl.c -index a8defc813c36..d3bb3865c1b1 100644 ---- a/arch/x86/events/rapl.c -+++ b/arch/x86/events/rapl.c -@@ -39,6 +39,10 @@ - * event: rapl_energy_psys - * perf code: 0x5 - * -+ * core counter: consumption of a single physical core -+ * event: rapl_energy_core (power_core PMU) -+ * perf code: 0x1 -+ * - * We manage those counters as free running (read-only). They may be - * use simultaneously by other tools, such as turbostat. - * -@@ -70,18 +74,22 @@ MODULE_LICENSE("GPL"); - /* - * RAPL energy status counters - */ --enum perf_rapl_events { -+enum perf_rapl_pkg_events { - PERF_RAPL_PP0 = 0, /* all cores */ - PERF_RAPL_PKG, /* entire package */ - PERF_RAPL_RAM, /* DRAM */ - PERF_RAPL_PP1, /* gpu */ - PERF_RAPL_PSYS, /* psys */ - -- PERF_RAPL_MAX, -- NR_RAPL_DOMAINS = PERF_RAPL_MAX, -+ PERF_RAPL_PKG_EVENTS_MAX, -+ NR_RAPL_PKG_DOMAINS = PERF_RAPL_PKG_EVENTS_MAX, - }; - --static const char *const rapl_domain_names[NR_RAPL_DOMAINS] __initconst = { -+#define PERF_RAPL_CORE 0 /* single core */ -+#define PERF_RAPL_CORE_EVENTS_MAX 1 -+#define NR_RAPL_CORE_DOMAINS PERF_RAPL_CORE_EVENTS_MAX -+ -+static const char *const rapl_pkg_domain_names[NR_RAPL_PKG_DOMAINS] __initconst = { - "pp0-core", - "package", - "dram", -@@ -89,6 +97,8 @@ static const char *const rapl_domain_names[NR_RAPL_DOMAINS] __initconst = { - "psys", - }; - -+static const char *const rapl_core_domain_name __initconst = "core"; -+ - /* - * event code: LSB 8 bits, passed in attr->config - * any other bit is reserved -@@ -112,7 +122,7 @@ static struct perf_pmu_events_attr event_attr_##v = { \ - * considered as either pkg-scope or die-scope, and we are considering - * them as die-scope. - */ --#define rapl_pmu_is_pkg_scope() \ -+#define rapl_pkg_pmu_is_pkg_scope() \ - (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || \ - boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) - -@@ -129,7 +139,8 @@ struct rapl_pmu { - struct rapl_pmus { - struct pmu pmu; - unsigned int nr_rapl_pmu; -- struct rapl_pmu *pmus[] __counted_by(nr_rapl_pmu); -+ unsigned int cntr_mask; -+ struct rapl_pmu *rapl_pmu[] __counted_by(nr_rapl_pmu); - }; - - enum rapl_unit_quirk { -@@ -139,44 +150,43 @@ enum rapl_unit_quirk { - }; - - struct rapl_model { -- struct perf_msr *rapl_msrs; -- unsigned long events; -+ struct perf_msr *rapl_pkg_msrs; -+ struct perf_msr *rapl_core_msrs; -+ unsigned long pkg_events; -+ unsigned long core_events; - unsigned int msr_power_unit; - enum rapl_unit_quirk unit_quirk; - }; - - /* 1/2^hw_unit Joule */ --static int rapl_hw_unit[NR_RAPL_DOMAINS] __read_mostly; --static struct rapl_pmus *rapl_pmus; --static unsigned int rapl_cntr_mask; -+static int rapl_pkg_hw_unit[NR_RAPL_PKG_DOMAINS] __read_mostly; -+static int rapl_core_hw_unit __read_mostly; -+static struct rapl_pmus *rapl_pmus_pkg; -+static struct rapl_pmus *rapl_pmus_core; - static u64 rapl_timer_ms; --static struct perf_msr *rapl_msrs; -+static struct rapl_model *rapl_model; - - /* -- * Helper functions to get the correct topology macros according to the -+ * Helper function to get the correct topology id according to the - * RAPL PMU scope. - */ --static inline unsigned int get_rapl_pmu_idx(int cpu) --{ -- return rapl_pmu_is_pkg_scope() ? topology_logical_package_id(cpu) : -- topology_logical_die_id(cpu); --} -- --static inline const struct cpumask *get_rapl_pmu_cpumask(int cpu) --{ -- return rapl_pmu_is_pkg_scope() ? topology_core_cpumask(cpu) : -- topology_die_cpumask(cpu); --} -- --static inline struct rapl_pmu *cpu_to_rapl_pmu(unsigned int cpu) -+static inline unsigned int get_rapl_pmu_idx(int cpu, int scope) - { -- unsigned int rapl_pmu_idx = get_rapl_pmu_idx(cpu); -- - /* -- * The unsigned check also catches the '-1' return value for non -- * existent mappings in the topology map. -+ * Returns unsigned int, which converts the '-1' return value -+ * (for non-existent mappings in topology map) to UINT_MAX, so -+ * the error check in the caller is simplified. - */ -- return rapl_pmu_idx < rapl_pmus->nr_rapl_pmu ? rapl_pmus->pmus[rapl_pmu_idx] : NULL; -+ switch (scope) { -+ case PERF_PMU_SCOPE_PKG: -+ return topology_logical_package_id(cpu); -+ case PERF_PMU_SCOPE_DIE: -+ return topology_logical_die_id(cpu); -+ case PERF_PMU_SCOPE_CORE: -+ return topology_logical_core_id(cpu); -+ default: -+ return -EINVAL; -+ } - } - - static inline u64 rapl_read_counter(struct perf_event *event) -@@ -186,19 +196,20 @@ static inline u64 rapl_read_counter(struct perf_event *event) - return raw; - } - --static inline u64 rapl_scale(u64 v, int cfg) -+static inline u64 rapl_scale(u64 v, struct perf_event *event) - { -- if (cfg > NR_RAPL_DOMAINS) { -- pr_warn("Invalid domain %d, failed to scale data\n", cfg); -- return v; -- } -+ int hw_unit = rapl_pkg_hw_unit[event->hw.config - 1]; -+ -+ if (event->pmu->scope == PERF_PMU_SCOPE_CORE) -+ hw_unit = rapl_core_hw_unit; -+ - /* - * scale delta to smallest unit (1/2^32) - * users must then scale back: count * 1/(1e9*2^32) to get Joules - * or use ldexp(count, -32). - * Watts = Joules/Time delta - */ -- return v << (32 - rapl_hw_unit[cfg - 1]); -+ return v << (32 - hw_unit); - } - - static u64 rapl_event_update(struct perf_event *event) -@@ -225,7 +236,7 @@ static u64 rapl_event_update(struct perf_event *event) - delta = (new_raw_count << shift) - (prev_raw_count << shift); - delta >>= shift; - -- sdelta = rapl_scale(delta, event->hw.config); -+ sdelta = rapl_scale(delta, event); - - local64_add(sdelta, &event->count); - -@@ -240,34 +251,34 @@ static void rapl_start_hrtimer(struct rapl_pmu *pmu) - - static enum hrtimer_restart rapl_hrtimer_handle(struct hrtimer *hrtimer) - { -- struct rapl_pmu *pmu = container_of(hrtimer, struct rapl_pmu, hrtimer); -+ struct rapl_pmu *rapl_pmu = container_of(hrtimer, struct rapl_pmu, hrtimer); - struct perf_event *event; - unsigned long flags; - -- if (!pmu->n_active) -+ if (!rapl_pmu->n_active) - return HRTIMER_NORESTART; - -- raw_spin_lock_irqsave(&pmu->lock, flags); -+ raw_spin_lock_irqsave(&rapl_pmu->lock, flags); - -- list_for_each_entry(event, &pmu->active_list, active_entry) -+ list_for_each_entry(event, &rapl_pmu->active_list, active_entry) - rapl_event_update(event); - -- raw_spin_unlock_irqrestore(&pmu->lock, flags); -+ raw_spin_unlock_irqrestore(&rapl_pmu->lock, flags); - -- hrtimer_forward_now(hrtimer, pmu->timer_interval); -+ hrtimer_forward_now(hrtimer, rapl_pmu->timer_interval); - - return HRTIMER_RESTART; - } - --static void rapl_hrtimer_init(struct rapl_pmu *pmu) -+static void rapl_hrtimer_init(struct rapl_pmu *rapl_pmu) - { -- struct hrtimer *hr = &pmu->hrtimer; -+ struct hrtimer *hr = &rapl_pmu->hrtimer; - - hrtimer_init(hr, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - hr->function = rapl_hrtimer_handle; - } - --static void __rapl_pmu_event_start(struct rapl_pmu *pmu, -+static void __rapl_pmu_event_start(struct rapl_pmu *rapl_pmu, - struct perf_event *event) - { - if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) -@@ -275,39 +286,39 @@ static void __rapl_pmu_event_start(struct rapl_pmu *pmu, - - event->hw.state = 0; - -- list_add_tail(&event->active_entry, &pmu->active_list); -+ list_add_tail(&event->active_entry, &rapl_pmu->active_list); - - local64_set(&event->hw.prev_count, rapl_read_counter(event)); - -- pmu->n_active++; -- if (pmu->n_active == 1) -- rapl_start_hrtimer(pmu); -+ rapl_pmu->n_active++; -+ if (rapl_pmu->n_active == 1) -+ rapl_start_hrtimer(rapl_pmu); - } - - static void rapl_pmu_event_start(struct perf_event *event, int mode) - { -- struct rapl_pmu *pmu = event->pmu_private; -+ struct rapl_pmu *rapl_pmu = event->pmu_private; - unsigned long flags; - -- raw_spin_lock_irqsave(&pmu->lock, flags); -- __rapl_pmu_event_start(pmu, event); -- raw_spin_unlock_irqrestore(&pmu->lock, flags); -+ raw_spin_lock_irqsave(&rapl_pmu->lock, flags); -+ __rapl_pmu_event_start(rapl_pmu, event); -+ raw_spin_unlock_irqrestore(&rapl_pmu->lock, flags); - } - - static void rapl_pmu_event_stop(struct perf_event *event, int mode) - { -- struct rapl_pmu *pmu = event->pmu_private; -+ struct rapl_pmu *rapl_pmu = event->pmu_private; - struct hw_perf_event *hwc = &event->hw; - unsigned long flags; - -- raw_spin_lock_irqsave(&pmu->lock, flags); -+ raw_spin_lock_irqsave(&rapl_pmu->lock, flags); - - /* mark event as deactivated and stopped */ - if (!(hwc->state & PERF_HES_STOPPED)) { -- WARN_ON_ONCE(pmu->n_active <= 0); -- pmu->n_active--; -- if (pmu->n_active == 0) -- hrtimer_cancel(&pmu->hrtimer); -+ WARN_ON_ONCE(rapl_pmu->n_active <= 0); -+ rapl_pmu->n_active--; -+ if (rapl_pmu->n_active == 0) -+ hrtimer_cancel(&rapl_pmu->hrtimer); - - list_del(&event->active_entry); - -@@ -325,23 +336,23 @@ static void rapl_pmu_event_stop(struct perf_event *event, int mode) - hwc->state |= PERF_HES_UPTODATE; - } - -- raw_spin_unlock_irqrestore(&pmu->lock, flags); -+ raw_spin_unlock_irqrestore(&rapl_pmu->lock, flags); - } - - static int rapl_pmu_event_add(struct perf_event *event, int mode) - { -- struct rapl_pmu *pmu = event->pmu_private; -+ struct rapl_pmu *rapl_pmu = event->pmu_private; - struct hw_perf_event *hwc = &event->hw; - unsigned long flags; - -- raw_spin_lock_irqsave(&pmu->lock, flags); -+ raw_spin_lock_irqsave(&rapl_pmu->lock, flags); - - hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; - - if (mode & PERF_EF_START) -- __rapl_pmu_event_start(pmu, event); -+ __rapl_pmu_event_start(rapl_pmu, event); - -- raw_spin_unlock_irqrestore(&pmu->lock, flags); -+ raw_spin_unlock_irqrestore(&rapl_pmu->lock, flags); - - return 0; - } -@@ -354,12 +365,14 @@ static void rapl_pmu_event_del(struct perf_event *event, int flags) - static int rapl_pmu_event_init(struct perf_event *event) - { - u64 cfg = event->attr.config & RAPL_EVENT_MASK; -- int bit, ret = 0; -- struct rapl_pmu *pmu; -+ int bit, rapl_pmus_scope, ret = 0; -+ struct rapl_pmu *rapl_pmu; -+ unsigned int rapl_pmu_idx; -+ struct rapl_pmus *rapl_pmus; - -- /* only look at RAPL events */ -- if (event->attr.type != rapl_pmus->pmu.type) -- return -ENOENT; -+ /* unsupported modes and filters */ -+ if (event->attr.sample_period) /* no sampling */ -+ return -EINVAL; - - /* check only supported bits are set */ - if (event->attr.config & ~RAPL_EVENT_MASK) -@@ -368,26 +381,49 @@ static int rapl_pmu_event_init(struct perf_event *event) - if (event->cpu < 0) - return -EINVAL; - -- if (!cfg || cfg >= NR_RAPL_DOMAINS + 1) -+ rapl_pmus = container_of(event->pmu, struct rapl_pmus, pmu); -+ if (!rapl_pmus) -+ return -EINVAL; -+ rapl_pmus_scope = rapl_pmus->pmu.scope; -+ -+ if (rapl_pmus_scope == PERF_PMU_SCOPE_PKG || rapl_pmus_scope == PERF_PMU_SCOPE_DIE) { -+ /* only look at RAPL package events */ -+ if (event->attr.type != rapl_pmus_pkg->pmu.type) -+ return -ENOENT; -+ -+ cfg = array_index_nospec((long)cfg, NR_RAPL_PKG_DOMAINS + 1); -+ if (!cfg || cfg >= NR_RAPL_PKG_DOMAINS + 1) -+ return -EINVAL; -+ -+ bit = cfg - 1; -+ event->hw.event_base = rapl_model->rapl_pkg_msrs[bit].msr; -+ } else if (rapl_pmus_scope == PERF_PMU_SCOPE_CORE) { -+ /* only look at RAPL core events */ -+ if (event->attr.type != rapl_pmus_core->pmu.type) -+ return -ENOENT; -+ -+ cfg = array_index_nospec((long)cfg, NR_RAPL_CORE_DOMAINS + 1); -+ if (!cfg || cfg >= NR_RAPL_PKG_DOMAINS + 1) -+ return -EINVAL; -+ -+ bit = cfg - 1; -+ event->hw.event_base = rapl_model->rapl_core_msrs[bit].msr; -+ } else - return -EINVAL; -- -- cfg = array_index_nospec((long)cfg, NR_RAPL_DOMAINS + 1); -- bit = cfg - 1; - - /* check event supported */ -- if (!(rapl_cntr_mask & (1 << bit))) -+ if (!(rapl_pmus->cntr_mask & (1 << bit))) - return -EINVAL; - -- /* unsupported modes and filters */ -- if (event->attr.sample_period) /* no sampling */ -+ rapl_pmu_idx = get_rapl_pmu_idx(event->cpu, rapl_pmus_scope); -+ if (rapl_pmu_idx >= rapl_pmus->nr_rapl_pmu) - return -EINVAL; -- - /* must be done before validate_group */ -- pmu = cpu_to_rapl_pmu(event->cpu); -- if (!pmu) -+ rapl_pmu = rapl_pmus->rapl_pmu[rapl_pmu_idx]; -+ if (!rapl_pmu) - return -EINVAL; -- event->pmu_private = pmu; -- event->hw.event_base = rapl_msrs[bit].msr; -+ -+ event->pmu_private = rapl_pmu; - event->hw.config = cfg; - event->hw.idx = bit; - -@@ -404,12 +440,14 @@ RAPL_EVENT_ATTR_STR(energy-pkg , rapl_pkg, "event=0x02"); - RAPL_EVENT_ATTR_STR(energy-ram , rapl_ram, "event=0x03"); - RAPL_EVENT_ATTR_STR(energy-gpu , rapl_gpu, "event=0x04"); - RAPL_EVENT_ATTR_STR(energy-psys, rapl_psys, "event=0x05"); -+RAPL_EVENT_ATTR_STR(energy-core, rapl_core, "event=0x01"); - - RAPL_EVENT_ATTR_STR(energy-cores.unit, rapl_cores_unit, "Joules"); - RAPL_EVENT_ATTR_STR(energy-pkg.unit , rapl_pkg_unit, "Joules"); - RAPL_EVENT_ATTR_STR(energy-ram.unit , rapl_ram_unit, "Joules"); - RAPL_EVENT_ATTR_STR(energy-gpu.unit , rapl_gpu_unit, "Joules"); - RAPL_EVENT_ATTR_STR(energy-psys.unit, rapl_psys_unit, "Joules"); -+RAPL_EVENT_ATTR_STR(energy-core.unit, rapl_core_unit, "Joules"); - - /* - * we compute in 0.23 nJ increments regardless of MSR -@@ -419,6 +457,7 @@ RAPL_EVENT_ATTR_STR(energy-pkg.scale, rapl_pkg_scale, "2.3283064365386962890 - RAPL_EVENT_ATTR_STR(energy-ram.scale, rapl_ram_scale, "2.3283064365386962890625e-10"); - RAPL_EVENT_ATTR_STR(energy-gpu.scale, rapl_gpu_scale, "2.3283064365386962890625e-10"); - RAPL_EVENT_ATTR_STR(energy-psys.scale, rapl_psys_scale, "2.3283064365386962890625e-10"); -+RAPL_EVENT_ATTR_STR(energy-core.scale, rapl_core_scale, "2.3283064365386962890625e-10"); - - /* - * There are no default events, but we need to create -@@ -451,6 +490,12 @@ static const struct attribute_group *rapl_attr_groups[] = { - NULL, - }; - -+static const struct attribute_group *rapl_core_attr_groups[] = { -+ &rapl_pmu_format_group, -+ &rapl_pmu_events_group, -+ NULL, -+}; -+ - static struct attribute *rapl_events_cores[] = { - EVENT_PTR(rapl_cores), - EVENT_PTR(rapl_cores_unit), -@@ -511,6 +556,18 @@ static struct attribute_group rapl_events_psys_group = { - .attrs = rapl_events_psys, - }; - -+static struct attribute *rapl_events_core[] = { -+ EVENT_PTR(rapl_core), -+ EVENT_PTR(rapl_core_unit), -+ EVENT_PTR(rapl_core_scale), -+ NULL, -+}; -+ -+static struct attribute_group rapl_events_core_group = { -+ .name = "events", -+ .attrs = rapl_events_core, -+}; -+ - static bool test_msr(int idx, void *data) - { - return test_bit(idx, (unsigned long *) data); -@@ -536,11 +593,11 @@ static struct perf_msr intel_rapl_spr_msrs[] = { - }; - - /* -- * Force to PERF_RAPL_MAX size due to: -- * - perf_msr_probe(PERF_RAPL_MAX) -+ * Force to PERF_RAPL_PKG_EVENTS_MAX size due to: -+ * - perf_msr_probe(PERF_RAPL_PKG_EVENTS_MAX) - * - want to use same event codes across both architectures - */ --static struct perf_msr amd_rapl_msrs[] = { -+static struct perf_msr amd_rapl_pkg_msrs[] = { - [PERF_RAPL_PP0] = { 0, &rapl_events_cores_group, NULL, false, 0 }, - [PERF_RAPL_PKG] = { MSR_AMD_PKG_ENERGY_STATUS, &rapl_events_pkg_group, test_msr, false, RAPL_MSR_MASK }, - [PERF_RAPL_RAM] = { 0, &rapl_events_ram_group, NULL, false, 0 }, -@@ -548,18 +605,25 @@ static struct perf_msr amd_rapl_msrs[] = { - [PERF_RAPL_PSYS] = { 0, &rapl_events_psys_group, NULL, false, 0 }, - }; - --static int rapl_check_hw_unit(struct rapl_model *rm) -+static struct perf_msr amd_rapl_core_msrs[] = { -+ [PERF_RAPL_CORE] = { MSR_AMD_CORE_ENERGY_STATUS, &rapl_events_core_group, -+ test_msr, false, RAPL_MSR_MASK }, -+}; -+ -+static int rapl_check_hw_unit(void) - { - u64 msr_rapl_power_unit_bits; - int i; - - /* protect rdmsrl() to handle virtualization */ -- if (rdmsrl_safe(rm->msr_power_unit, &msr_rapl_power_unit_bits)) -+ if (rdmsrl_safe(rapl_model->msr_power_unit, &msr_rapl_power_unit_bits)) - return -1; -- for (i = 0; i < NR_RAPL_DOMAINS; i++) -- rapl_hw_unit[i] = (msr_rapl_power_unit_bits >> 8) & 0x1FULL; -+ for (i = 0; i < NR_RAPL_PKG_DOMAINS; i++) -+ rapl_pkg_hw_unit[i] = (msr_rapl_power_unit_bits >> 8) & 0x1FULL; - -- switch (rm->unit_quirk) { -+ rapl_core_hw_unit = (msr_rapl_power_unit_bits >> 8) & 0x1FULL; -+ -+ switch (rapl_model->unit_quirk) { - /* - * DRAM domain on HSW server and KNL has fixed energy unit which can be - * different than the unit from power unit MSR. See -@@ -567,17 +631,16 @@ static int rapl_check_hw_unit(struct rapl_model *rm) - * of 2. Datasheet, September 2014, Reference Number: 330784-001 " - */ - case RAPL_UNIT_QUIRK_INTEL_HSW: -- rapl_hw_unit[PERF_RAPL_RAM] = 16; -+ rapl_pkg_hw_unit[PERF_RAPL_RAM] = 16; - break; - /* SPR uses a fixed energy unit for Psys domain. */ - case RAPL_UNIT_QUIRK_INTEL_SPR: -- rapl_hw_unit[PERF_RAPL_PSYS] = 0; -+ rapl_pkg_hw_unit[PERF_RAPL_PSYS] = 0; - break; - default: - break; - } - -- - /* - * Calculate the timer rate: - * Use reference of 200W for scaling the timeout to avoid counter -@@ -586,9 +649,9 @@ static int rapl_check_hw_unit(struct rapl_model *rm) - * if hw unit is 32, then we use 2 ms 1/200/2 - */ - rapl_timer_ms = 2; -- if (rapl_hw_unit[0] < 32) { -+ if (rapl_pkg_hw_unit[0] < 32) { - rapl_timer_ms = (1000 / (2 * 100)); -- rapl_timer_ms *= (1ULL << (32 - rapl_hw_unit[0] - 1)); -+ rapl_timer_ms *= (1ULL << (32 - rapl_pkg_hw_unit[0] - 1)); - } - return 0; - } -@@ -596,24 +659,32 @@ static int rapl_check_hw_unit(struct rapl_model *rm) - static void __init rapl_advertise(void) - { - int i; -+ int num_counters = hweight32(rapl_pmus_pkg->cntr_mask); -+ -+ if (rapl_pmus_core) -+ num_counters += hweight32(rapl_pmus_core->cntr_mask); - - pr_info("API unit is 2^-32 Joules, %d fixed counters, %llu ms ovfl timer\n", -- hweight32(rapl_cntr_mask), rapl_timer_ms); -+ num_counters, rapl_timer_ms); - -- for (i = 0; i < NR_RAPL_DOMAINS; i++) { -- if (rapl_cntr_mask & (1 << i)) { -+ for (i = 0; i < NR_RAPL_PKG_DOMAINS; i++) { -+ if (rapl_pmus_pkg->cntr_mask & (1 << i)) { - pr_info("hw unit of domain %s 2^-%d Joules\n", -- rapl_domain_names[i], rapl_hw_unit[i]); -+ rapl_pkg_domain_names[i], rapl_pkg_hw_unit[i]); - } - } -+ -+ if (rapl_pmus_core && (rapl_pmus_core->cntr_mask & (1 << PERF_RAPL_CORE))) -+ pr_info("hw unit of domain %s 2^-%d Joules\n", -+ rapl_core_domain_name, rapl_core_hw_unit); - } - --static void cleanup_rapl_pmus(void) -+static void cleanup_rapl_pmus(struct rapl_pmus *rapl_pmus) - { - int i; - - for (i = 0; i < rapl_pmus->nr_rapl_pmu; i++) -- kfree(rapl_pmus->pmus[i]); -+ kfree(rapl_pmus->rapl_pmu[i]); - kfree(rapl_pmus); - } - -@@ -626,46 +697,60 @@ static const struct attribute_group *rapl_attr_update[] = { - NULL, - }; - --static int __init init_rapl_pmu(void) -+static const struct attribute_group *rapl_core_attr_update[] = { -+ &rapl_events_core_group, -+ NULL, -+}; -+ -+static int __init init_rapl_pmu(struct rapl_pmus *rapl_pmus) - { -- struct rapl_pmu *pmu; -+ struct rapl_pmu *rapl_pmu; - int idx; - - for (idx = 0; idx < rapl_pmus->nr_rapl_pmu; idx++) { -- pmu = kzalloc(sizeof(*pmu), GFP_KERNEL); -- if (!pmu) -+ rapl_pmu = kzalloc(sizeof(*rapl_pmu), GFP_KERNEL); -+ if (!rapl_pmu) - goto free; - -- raw_spin_lock_init(&pmu->lock); -- INIT_LIST_HEAD(&pmu->active_list); -- pmu->pmu = &rapl_pmus->pmu; -- pmu->timer_interval = ms_to_ktime(rapl_timer_ms); -- rapl_hrtimer_init(pmu); -+ raw_spin_lock_init(&rapl_pmu->lock); -+ INIT_LIST_HEAD(&rapl_pmu->active_list); -+ rapl_pmu->pmu = &rapl_pmus->pmu; -+ rapl_pmu->timer_interval = ms_to_ktime(rapl_timer_ms); -+ rapl_hrtimer_init(rapl_pmu); - -- rapl_pmus->pmus[idx] = pmu; -+ rapl_pmus->rapl_pmu[idx] = rapl_pmu; - } - - return 0; - free: - for (; idx > 0; idx--) -- kfree(rapl_pmus->pmus[idx - 1]); -+ kfree(rapl_pmus->rapl_pmu[idx - 1]); - return -ENOMEM; - } - --static int __init init_rapl_pmus(void) -+static int __init init_rapl_pmus(struct rapl_pmus **rapl_pmus_ptr, int rapl_pmu_scope, -+ const struct attribute_group **rapl_attr_groups, -+ const struct attribute_group **rapl_attr_update) - { - int nr_rapl_pmu = topology_max_packages(); -- int rapl_pmu_scope = PERF_PMU_SCOPE_PKG; -+ struct rapl_pmus *rapl_pmus; - -- if (!rapl_pmu_is_pkg_scope()) { -- nr_rapl_pmu *= topology_max_dies_per_package(); -- rapl_pmu_scope = PERF_PMU_SCOPE_DIE; -- } -+ /* -+ * rapl_pmu_scope must be either PKG, DIE or CORE -+ */ -+ if (rapl_pmu_scope == PERF_PMU_SCOPE_DIE) -+ nr_rapl_pmu *= topology_max_dies_per_package(); -+ else if (rapl_pmu_scope == PERF_PMU_SCOPE_CORE) -+ nr_rapl_pmu *= topology_num_cores_per_package(); -+ else if (rapl_pmu_scope != PERF_PMU_SCOPE_PKG) -+ return -EINVAL; - -- rapl_pmus = kzalloc(struct_size(rapl_pmus, pmus, nr_rapl_pmu), GFP_KERNEL); -+ rapl_pmus = kzalloc(struct_size(rapl_pmus, rapl_pmu, nr_rapl_pmu), GFP_KERNEL); - if (!rapl_pmus) - return -ENOMEM; - -+ *rapl_pmus_ptr = rapl_pmus; -+ - rapl_pmus->nr_rapl_pmu = nr_rapl_pmu; - rapl_pmus->pmu.attr_groups = rapl_attr_groups; - rapl_pmus->pmu.attr_update = rapl_attr_update; -@@ -680,75 +765,77 @@ static int __init init_rapl_pmus(void) - rapl_pmus->pmu.module = THIS_MODULE; - rapl_pmus->pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE; - -- return init_rapl_pmu(); -+ return init_rapl_pmu(rapl_pmus); - } - - static struct rapl_model model_snb = { -- .events = BIT(PERF_RAPL_PP0) | -+ .pkg_events = BIT(PERF_RAPL_PP0) | - BIT(PERF_RAPL_PKG) | - BIT(PERF_RAPL_PP1), - .msr_power_unit = MSR_RAPL_POWER_UNIT, -- .rapl_msrs = intel_rapl_msrs, -+ .rapl_pkg_msrs = intel_rapl_msrs, - }; - - static struct rapl_model model_snbep = { -- .events = BIT(PERF_RAPL_PP0) | -+ .pkg_events = BIT(PERF_RAPL_PP0) | - BIT(PERF_RAPL_PKG) | - BIT(PERF_RAPL_RAM), - .msr_power_unit = MSR_RAPL_POWER_UNIT, -- .rapl_msrs = intel_rapl_msrs, -+ .rapl_pkg_msrs = intel_rapl_msrs, - }; - - static struct rapl_model model_hsw = { -- .events = BIT(PERF_RAPL_PP0) | -+ .pkg_events = BIT(PERF_RAPL_PP0) | - BIT(PERF_RAPL_PKG) | - BIT(PERF_RAPL_RAM) | - BIT(PERF_RAPL_PP1), - .msr_power_unit = MSR_RAPL_POWER_UNIT, -- .rapl_msrs = intel_rapl_msrs, -+ .rapl_pkg_msrs = intel_rapl_msrs, - }; - - static struct rapl_model model_hsx = { -- .events = BIT(PERF_RAPL_PP0) | -+ .pkg_events = BIT(PERF_RAPL_PP0) | - BIT(PERF_RAPL_PKG) | - BIT(PERF_RAPL_RAM), - .unit_quirk = RAPL_UNIT_QUIRK_INTEL_HSW, - .msr_power_unit = MSR_RAPL_POWER_UNIT, -- .rapl_msrs = intel_rapl_msrs, -+ .rapl_pkg_msrs = intel_rapl_msrs, - }; - - static struct rapl_model model_knl = { -- .events = BIT(PERF_RAPL_PKG) | -+ .pkg_events = BIT(PERF_RAPL_PKG) | - BIT(PERF_RAPL_RAM), - .unit_quirk = RAPL_UNIT_QUIRK_INTEL_HSW, - .msr_power_unit = MSR_RAPL_POWER_UNIT, -- .rapl_msrs = intel_rapl_msrs, -+ .rapl_pkg_msrs = intel_rapl_msrs, - }; - - static struct rapl_model model_skl = { -- .events = BIT(PERF_RAPL_PP0) | -+ .pkg_events = BIT(PERF_RAPL_PP0) | - BIT(PERF_RAPL_PKG) | - BIT(PERF_RAPL_RAM) | - BIT(PERF_RAPL_PP1) | - BIT(PERF_RAPL_PSYS), - .msr_power_unit = MSR_RAPL_POWER_UNIT, -- .rapl_msrs = intel_rapl_msrs, -+ .rapl_pkg_msrs = intel_rapl_msrs, - }; - - static struct rapl_model model_spr = { -- .events = BIT(PERF_RAPL_PP0) | -+ .pkg_events = BIT(PERF_RAPL_PP0) | - BIT(PERF_RAPL_PKG) | - BIT(PERF_RAPL_RAM) | - BIT(PERF_RAPL_PSYS), - .unit_quirk = RAPL_UNIT_QUIRK_INTEL_SPR, - .msr_power_unit = MSR_RAPL_POWER_UNIT, -- .rapl_msrs = intel_rapl_spr_msrs, -+ .rapl_pkg_msrs = intel_rapl_spr_msrs, - }; - - static struct rapl_model model_amd_hygon = { -- .events = BIT(PERF_RAPL_PKG), -+ .pkg_events = BIT(PERF_RAPL_PKG), -+ .core_events = BIT(PERF_RAPL_CORE), - .msr_power_unit = MSR_AMD_RAPL_POWER_UNIT, -- .rapl_msrs = amd_rapl_msrs, -+ .rapl_pkg_msrs = amd_rapl_pkg_msrs, -+ .rapl_core_msrs = amd_rapl_core_msrs, - }; - - static const struct x86_cpu_id rapl_model_match[] __initconst = { -@@ -804,45 +891,73 @@ MODULE_DEVICE_TABLE(x86cpu, rapl_model_match); - static int __init rapl_pmu_init(void) - { - const struct x86_cpu_id *id; -- struct rapl_model *rm; -+ int rapl_pkg_pmu_scope = PERF_PMU_SCOPE_DIE; - int ret; - -+ if (rapl_pkg_pmu_is_pkg_scope()) -+ rapl_pkg_pmu_scope = PERF_PMU_SCOPE_PKG; -+ - id = x86_match_cpu(rapl_model_match); - if (!id) - return -ENODEV; - -- rm = (struct rapl_model *) id->driver_data; -- -- rapl_msrs = rm->rapl_msrs; -+ rapl_model = (struct rapl_model *) id->driver_data; - -- rapl_cntr_mask = perf_msr_probe(rapl_msrs, PERF_RAPL_MAX, -- false, (void *) &rm->events); -- -- ret = rapl_check_hw_unit(rm); -+ ret = rapl_check_hw_unit(); - if (ret) - return ret; - -- ret = init_rapl_pmus(); -+ ret = init_rapl_pmus(&rapl_pmus_pkg, rapl_pkg_pmu_scope, rapl_attr_groups, -+ rapl_attr_update); - if (ret) - return ret; - -- ret = perf_pmu_register(&rapl_pmus->pmu, "power", -1); -+ rapl_pmus_pkg->cntr_mask = perf_msr_probe(rapl_model->rapl_pkg_msrs, -+ PERF_RAPL_PKG_EVENTS_MAX, false, -+ (void *) &rapl_model->pkg_events); -+ -+ ret = perf_pmu_register(&rapl_pmus_pkg->pmu, "power", -1); - if (ret) - goto out; - -+ if (rapl_model->core_events) { -+ ret = init_rapl_pmus(&rapl_pmus_core, PERF_PMU_SCOPE_CORE, -+ rapl_core_attr_groups, -+ rapl_core_attr_update); -+ if (ret) { -+ pr_warn("power-core PMU initialization failed (%d)\n", ret); -+ goto core_init_failed; -+ } -+ -+ rapl_pmus_core->cntr_mask = perf_msr_probe(rapl_model->rapl_core_msrs, -+ PERF_RAPL_CORE_EVENTS_MAX, false, -+ (void *) &rapl_model->core_events); -+ -+ ret = perf_pmu_register(&rapl_pmus_core->pmu, "power_core", -1); -+ if (ret) { -+ pr_warn("power-core PMU registration failed (%d)\n", ret); -+ cleanup_rapl_pmus(rapl_pmus_core); -+ } -+ } -+ -+core_init_failed: - rapl_advertise(); - return 0; - - out: - pr_warn("Initialization failed (%d), disabled\n", ret); -- cleanup_rapl_pmus(); -+ cleanup_rapl_pmus(rapl_pmus_pkg); - return ret; - } - module_init(rapl_pmu_init); - - static void __exit intel_rapl_exit(void) - { -- perf_pmu_unregister(&rapl_pmus->pmu); -- cleanup_rapl_pmus(); -+ if (rapl_pmus_core) { -+ perf_pmu_unregister(&rapl_pmus_core->pmu); -+ cleanup_rapl_pmus(rapl_pmus_core); -+ } -+ perf_pmu_unregister(&rapl_pmus_pkg->pmu); -+ cleanup_rapl_pmus(rapl_pmus_pkg); - } - module_exit(intel_rapl_exit); -diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h -index 20e6009381ed..c0cd10182e90 100644 ---- a/arch/x86/include/asm/processor.h -+++ b/arch/x86/include/asm/processor.h -@@ -98,6 +98,7 @@ struct cpuinfo_topology { - // Logical ID mappings - u32 logical_pkg_id; - u32 logical_die_id; -+ u32 logical_core_id; - - // AMD Node ID and Nodes per Package info - u32 amd_node_id; -diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h -index 63bab25a4896..ec134b719144 100644 ---- a/arch/x86/include/asm/topology.h -+++ b/arch/x86/include/asm/topology.h -@@ -143,6 +143,7 @@ extern const struct cpumask *cpu_clustergroup_mask(int cpu); - #define topology_logical_package_id(cpu) (cpu_data(cpu).topo.logical_pkg_id) - #define topology_physical_package_id(cpu) (cpu_data(cpu).topo.pkg_id) - #define topology_logical_die_id(cpu) (cpu_data(cpu).topo.logical_die_id) -+#define topology_logical_core_id(cpu) (cpu_data(cpu).topo.logical_core_id) - #define topology_die_id(cpu) (cpu_data(cpu).topo.die_id) - #define topology_core_id(cpu) (cpu_data(cpu).topo.core_id) - #define topology_ppin(cpu) (cpu_data(cpu).ppin) -diff --git a/arch/x86/kernel/cpu/debugfs.c b/arch/x86/kernel/cpu/debugfs.c -index 10719aba6276..cacfd3f6abef 100644 ---- a/arch/x86/kernel/cpu/debugfs.c -+++ b/arch/x86/kernel/cpu/debugfs.c -@@ -25,6 +25,7 @@ static int cpu_debug_show(struct seq_file *m, void *p) - seq_printf(m, "cpu_type: %s\n", get_topology_cpu_type_name(c)); - seq_printf(m, "logical_pkg_id: %u\n", c->topo.logical_pkg_id); - seq_printf(m, "logical_die_id: %u\n", c->topo.logical_die_id); -+ seq_printf(m, "logical_core_id: %u\n", c->topo.logical_core_id); - seq_printf(m, "llc_id: %u\n", c->topo.llc_id); - seq_printf(m, "l2c_id: %u\n", c->topo.l2c_id); - seq_printf(m, "amd_node_id: %u\n", c->topo.amd_node_id); -diff --git a/arch/x86/kernel/cpu/topology_common.c b/arch/x86/kernel/cpu/topology_common.c -index 8277c64f88db..b5a5e1411469 100644 ---- a/arch/x86/kernel/cpu/topology_common.c -+++ b/arch/x86/kernel/cpu/topology_common.c -@@ -185,6 +185,7 @@ static void topo_set_ids(struct topo_scan *tscan, bool early) - if (!early) { - c->topo.logical_pkg_id = topology_get_logical_id(apicid, TOPO_PKG_DOMAIN); - c->topo.logical_die_id = topology_get_logical_id(apicid, TOPO_DIE_DOMAIN); -+ c->topo.logical_core_id = topology_get_logical_id(apicid, TOPO_CORE_DOMAIN); - } - - /* Package relative core ID */ --- -2.48.0.rc1 - -From da4f2e91d18239b158f7f48b0bd718110ade01ff Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Sun, 2 Feb 2025 13:55:04 +0100 -Subject: [PATCH 10/12] pksm - -Signed-off-by: Peter Jung ---- - arch/alpha/kernel/syscalls/syscall.tbl | 3 + - arch/arm/tools/syscall.tbl | 3 + - arch/m68k/kernel/syscalls/syscall.tbl | 3 + - arch/microblaze/kernel/syscalls/syscall.tbl | 3 + - arch/mips/kernel/syscalls/syscall_n32.tbl | 3 + - arch/mips/kernel/syscalls/syscall_n64.tbl | 3 + - arch/mips/kernel/syscalls/syscall_o32.tbl | 3 + - arch/parisc/kernel/syscalls/syscall.tbl | 3 + - arch/powerpc/kernel/syscalls/syscall.tbl | 3 + - arch/s390/kernel/syscalls/syscall.tbl | 3 + - arch/sh/kernel/syscalls/syscall.tbl | 3 + - arch/sparc/kernel/syscalls/syscall.tbl | 3 + - arch/x86/entry/syscalls/syscall_32.tbl | 3 + - arch/x86/entry/syscalls/syscall_64.tbl | 3 + - arch/xtensa/kernel/syscalls/syscall.tbl | 3 + - include/linux/syscalls.h | 3 + - include/uapi/asm-generic/unistd.h | 9 +- - kernel/sys.c | 138 ++++++++++++++++++ - kernel/sys_ni.c | 3 + - scripts/syscall.tbl | 3 + - .../arch/powerpc/entry/syscalls/syscall.tbl | 3 + - .../perf/arch/s390/entry/syscalls/syscall.tbl | 3 + - 22 files changed, 206 insertions(+), 1 deletion(-) - -diff --git a/arch/alpha/kernel/syscalls/syscall.tbl b/arch/alpha/kernel/syscalls/syscall.tbl -index c59d53d6d3f3..121696f903e8 100644 ---- a/arch/alpha/kernel/syscalls/syscall.tbl -+++ b/arch/alpha/kernel/syscalls/syscall.tbl -@@ -506,3 +506,6 @@ - 574 common getxattrat sys_getxattrat - 575 common listxattrat sys_listxattrat - 576 common removexattrat sys_removexattrat -+577 common process_ksm_enable sys_process_ksm_enable -+578 common process_ksm_disable sys_process_ksm_disable -+579 common process_ksm_status sys_process_ksm_status -diff --git a/arch/arm/tools/syscall.tbl b/arch/arm/tools/syscall.tbl -index 49eeb2ad8dbd..1ce4d983b5b2 100644 ---- a/arch/arm/tools/syscall.tbl -+++ b/arch/arm/tools/syscall.tbl -@@ -481,3 +481,6 @@ - 464 common getxattrat sys_getxattrat - 465 common listxattrat sys_listxattrat - 466 common removexattrat sys_removexattrat -+467 common process_ksm_enable sys_process_ksm_enable -+468 common process_ksm_disable sys_process_ksm_disable -+469 common process_ksm_status sys_process_ksm_status -diff --git a/arch/m68k/kernel/syscalls/syscall.tbl b/arch/m68k/kernel/syscalls/syscall.tbl -index f5ed71f1910d..17e865370d37 100644 ---- a/arch/m68k/kernel/syscalls/syscall.tbl -+++ b/arch/m68k/kernel/syscalls/syscall.tbl -@@ -466,3 +466,6 @@ - 464 common getxattrat sys_getxattrat - 465 common listxattrat sys_listxattrat - 466 common removexattrat sys_removexattrat -+467 common process_ksm_enable sys_process_ksm_enable -+468 common process_ksm_disable sys_process_ksm_disable -+469 common process_ksm_status sys_process_ksm_status -diff --git a/arch/microblaze/kernel/syscalls/syscall.tbl b/arch/microblaze/kernel/syscalls/syscall.tbl -index 680f568b77f2..64740e895587 100644 ---- a/arch/microblaze/kernel/syscalls/syscall.tbl -+++ b/arch/microblaze/kernel/syscalls/syscall.tbl -@@ -472,3 +472,6 @@ - 464 common getxattrat sys_getxattrat - 465 common listxattrat sys_listxattrat - 466 common removexattrat sys_removexattrat -+467 common process_ksm_enable sys_process_ksm_enable -+468 common process_ksm_disable sys_process_ksm_disable -+469 common process_ksm_status sys_process_ksm_status -diff --git a/arch/mips/kernel/syscalls/syscall_n32.tbl b/arch/mips/kernel/syscalls/syscall_n32.tbl -index 0b9b7e25b69a..bfafb91a2eda 100644 ---- a/arch/mips/kernel/syscalls/syscall_n32.tbl -+++ b/arch/mips/kernel/syscalls/syscall_n32.tbl -@@ -405,3 +405,6 @@ - 464 n32 getxattrat sys_getxattrat - 465 n32 listxattrat sys_listxattrat - 466 n32 removexattrat sys_removexattrat -+467 n32 process_ksm_enable sys_process_ksm_enable -+468 n32 process_ksm_disable sys_process_ksm_disable -+469 n32 process_ksm_status sys_process_ksm_status -diff --git a/arch/mips/kernel/syscalls/syscall_n64.tbl b/arch/mips/kernel/syscalls/syscall_n64.tbl -index c844cd5cda62..39d446aeac64 100644 ---- a/arch/mips/kernel/syscalls/syscall_n64.tbl -+++ b/arch/mips/kernel/syscalls/syscall_n64.tbl -@@ -381,3 +381,6 @@ - 464 n64 getxattrat sys_getxattrat - 465 n64 listxattrat sys_listxattrat - 466 n64 removexattrat sys_removexattrat -+467 n64 process_ksm_enable sys_process_ksm_enable -+468 n64 process_ksm_disable sys_process_ksm_disable -+469 n64 process_ksm_status sys_process_ksm_status -diff --git a/arch/mips/kernel/syscalls/syscall_o32.tbl b/arch/mips/kernel/syscalls/syscall_o32.tbl -index 349b8aad1159..61536c55715a 100644 ---- a/arch/mips/kernel/syscalls/syscall_o32.tbl -+++ b/arch/mips/kernel/syscalls/syscall_o32.tbl -@@ -454,3 +454,6 @@ - 464 o32 getxattrat sys_getxattrat - 465 o32 listxattrat sys_listxattrat - 466 o32 removexattrat sys_removexattrat -+467 o32 process_ksm_enable sys_process_ksm_enable -+468 o32 process_ksm_disable sys_process_ksm_disable -+469 o32 process_ksm_status sys_process_ksm_status -diff --git a/arch/parisc/kernel/syscalls/syscall.tbl b/arch/parisc/kernel/syscalls/syscall.tbl -index d9fc94c86965..85dca5afcf06 100644 ---- a/arch/parisc/kernel/syscalls/syscall.tbl -+++ b/arch/parisc/kernel/syscalls/syscall.tbl -@@ -465,3 +465,6 @@ - 464 common getxattrat sys_getxattrat - 465 common listxattrat sys_listxattrat - 466 common removexattrat sys_removexattrat -+467 common process_ksm_enable sys_process_ksm_enable -+468 common process_ksm_disable sys_process_ksm_disable -+469 common process_ksm_status sys_process_ksm_status -diff --git a/arch/powerpc/kernel/syscalls/syscall.tbl b/arch/powerpc/kernel/syscalls/syscall.tbl -index d8b4ab78bef0..57aa958c1b97 100644 ---- a/arch/powerpc/kernel/syscalls/syscall.tbl -+++ b/arch/powerpc/kernel/syscalls/syscall.tbl -@@ -557,3 +557,6 @@ - 464 common getxattrat sys_getxattrat - 465 common listxattrat sys_listxattrat - 466 common removexattrat sys_removexattrat -+467 common process_ksm_enable sys_process_ksm_enable -+468 common process_ksm_disable sys_process_ksm_disable -+469 common process_ksm_status sys_process_ksm_status -diff --git a/arch/s390/kernel/syscalls/syscall.tbl b/arch/s390/kernel/syscalls/syscall.tbl -index e9115b4d8b63..2afc778f2d17 100644 ---- a/arch/s390/kernel/syscalls/syscall.tbl -+++ b/arch/s390/kernel/syscalls/syscall.tbl -@@ -469,3 +469,6 @@ - 464 common getxattrat sys_getxattrat sys_getxattrat - 465 common listxattrat sys_listxattrat sys_listxattrat - 466 common removexattrat sys_removexattrat sys_removexattrat -+467 common process_ksm_enable sys_process_ksm_enable sys_process_ksm_enable -+468 common process_ksm_disable sys_process_ksm_disable sys_process_ksm_disable -+469 common process_ksm_status sys_process_ksm_status sys_process_ksm_status -diff --git a/arch/sh/kernel/syscalls/syscall.tbl b/arch/sh/kernel/syscalls/syscall.tbl -index c8cad33bf250..dfe06a84d902 100644 ---- a/arch/sh/kernel/syscalls/syscall.tbl -+++ b/arch/sh/kernel/syscalls/syscall.tbl -@@ -470,3 +470,6 @@ - 464 common getxattrat sys_getxattrat - 465 common listxattrat sys_listxattrat - 466 common removexattrat sys_removexattrat -+467 common process_ksm_enable sys_process_ksm_enable -+468 common process_ksm_disable sys_process_ksm_disable -+469 common process_ksm_status sys_process_ksm_status -diff --git a/arch/sparc/kernel/syscalls/syscall.tbl b/arch/sparc/kernel/syscalls/syscall.tbl -index 727f99d333b3..4c43b0d2d09f 100644 ---- a/arch/sparc/kernel/syscalls/syscall.tbl -+++ b/arch/sparc/kernel/syscalls/syscall.tbl -@@ -512,3 +512,6 @@ - 464 common getxattrat sys_getxattrat - 465 common listxattrat sys_listxattrat - 466 common removexattrat sys_removexattrat -+467 common process_ksm_enable sys_process_ksm_enable -+468 common process_ksm_disable sys_process_ksm_disable -+469 common process_ksm_status sys_process_ksm_status -diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl -index 4d0fb2fba7e2..a63252b84261 100644 ---- a/arch/x86/entry/syscalls/syscall_32.tbl -+++ b/arch/x86/entry/syscalls/syscall_32.tbl -@@ -472,3 +472,6 @@ - 464 i386 getxattrat sys_getxattrat - 465 i386 listxattrat sys_listxattrat - 466 i386 removexattrat sys_removexattrat -+467 i386 process_ksm_enable sys_process_ksm_enable -+468 i386 process_ksm_disable sys_process_ksm_disable -+469 i386 process_ksm_status sys_process_ksm_status -diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl -index 5eb708bff1c7..b5fe77405938 100644 ---- a/arch/x86/entry/syscalls/syscall_64.tbl -+++ b/arch/x86/entry/syscalls/syscall_64.tbl -@@ -390,6 +390,9 @@ - 464 common getxattrat sys_getxattrat - 465 common listxattrat sys_listxattrat - 466 common removexattrat sys_removexattrat -+467 common process_ksm_enable sys_process_ksm_enable -+468 common process_ksm_disable sys_process_ksm_disable -+469 common process_ksm_status sys_process_ksm_status - - # - # Due to a historical design error, certain syscalls are numbered differently -diff --git a/arch/xtensa/kernel/syscalls/syscall.tbl b/arch/xtensa/kernel/syscalls/syscall.tbl -index 37effc1b134e..5c944f0dcc20 100644 ---- a/arch/xtensa/kernel/syscalls/syscall.tbl -+++ b/arch/xtensa/kernel/syscalls/syscall.tbl -@@ -437,3 +437,6 @@ - 464 common getxattrat sys_getxattrat - 465 common listxattrat sys_listxattrat - 466 common removexattrat sys_removexattrat -+467 common process_ksm_enable sys_process_ksm_enable -+468 common process_ksm_disable sys_process_ksm_disable -+469 common process_ksm_status sys_process_ksm_status -diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h -index c6333204d451..00400d99eef3 100644 ---- a/include/linux/syscalls.h -+++ b/include/linux/syscalls.h -@@ -831,6 +831,9 @@ asmlinkage long sys_madvise(unsigned long start, size_t len, int behavior); - asmlinkage long sys_process_madvise(int pidfd, const struct iovec __user *vec, - size_t vlen, int behavior, unsigned int flags); - asmlinkage long sys_process_mrelease(int pidfd, unsigned int flags); -+asmlinkage long sys_process_ksm_enable(int pidfd, unsigned int flags); -+asmlinkage long sys_process_ksm_disable(int pidfd, unsigned int flags); -+asmlinkage long sys_process_ksm_status(int pidfd, unsigned int flags); - asmlinkage long sys_remap_file_pages(unsigned long start, unsigned long size, - unsigned long prot, unsigned long pgoff, - unsigned long flags); -diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h -index 88dc393c2bca..34d73f16b478 100644 ---- a/include/uapi/asm-generic/unistd.h -+++ b/include/uapi/asm-generic/unistd.h -@@ -850,8 +850,15 @@ __SYSCALL(__NR_listxattrat, sys_listxattrat) - #define __NR_removexattrat 466 - __SYSCALL(__NR_removexattrat, sys_removexattrat) - -+#define __NR_process_ksm_enable 467 -+__SYSCALL(__NR_process_ksm_enable, sys_process_ksm_enable) -+#define __NR_process_ksm_disable 468 -+__SYSCALL(__NR_process_ksm_disable, sys_process_ksm_disable) -+#define __NR_process_ksm_status 469 -+__SYSCALL(__NR_process_ksm_status, sys_process_ksm_status) -+ - #undef __NR_syscalls --#define __NR_syscalls 467 -+#define __NR_syscalls 470 - - /* - * 32 bit systems traditionally used different -diff --git a/kernel/sys.c b/kernel/sys.c -index c4c701c6f0b4..8806d113f5db 100644 ---- a/kernel/sys.c -+++ b/kernel/sys.c -@@ -2816,6 +2816,144 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, - return error; - } - -+#ifdef CONFIG_KSM -+enum pkc_action { -+ PKSM_ENABLE = 0, -+ PKSM_DISABLE, -+ PKSM_STATUS, -+}; -+ -+static long do_process_ksm_control(int pidfd, enum pkc_action action) -+{ -+ long ret; -+ struct task_struct *task; -+ struct mm_struct *mm; -+ unsigned int f_flags; -+ -+ task = pidfd_get_task(pidfd, &f_flags); -+ if (IS_ERR(task)) { -+ ret = PTR_ERR(task); -+ goto out; -+ } -+ -+ /* Require PTRACE_MODE_READ to avoid leaking ASLR metadata. */ -+ mm = mm_access(task, PTRACE_MODE_READ_FSCREDS); -+ if (IS_ERR_OR_NULL(mm)) { -+ ret = IS_ERR(mm) ? PTR_ERR(mm) : -ESRCH; -+ goto release_task; -+ } -+ -+ /* Require CAP_SYS_NICE for influencing process performance. */ -+ if (!capable(CAP_SYS_NICE)) { -+ ret = -EPERM; -+ goto release_mm; -+ } -+ -+ if (mmap_write_lock_killable(mm)) { -+ ret = -EINTR; -+ goto release_mm; -+ } -+ -+ switch (action) { -+ case PKSM_ENABLE: -+ ret = ksm_enable_merge_any(mm); -+ break; -+ case PKSM_DISABLE: -+ ret = ksm_disable_merge_any(mm); -+ break; -+ case PKSM_STATUS: -+ ret = !!test_bit(MMF_VM_MERGE_ANY, &mm->flags); -+ break; -+ } -+ -+ mmap_write_unlock(mm); -+ -+release_mm: -+ mmput(mm); -+release_task: -+ put_task_struct(task); -+out: -+ return ret; -+} -+#endif /* CONFIG_KSM */ -+ -+SYSCALL_DEFINE2(process_ksm_enable, int, pidfd, unsigned int, flags) -+{ -+#ifdef CONFIG_KSM -+ if (flags != 0) -+ return -EINVAL; -+ -+ return do_process_ksm_control(pidfd, PKSM_ENABLE); -+#else /* CONFIG_KSM */ -+ return -ENOSYS; -+#endif /* CONFIG_KSM */ -+} -+ -+SYSCALL_DEFINE2(process_ksm_disable, int, pidfd, unsigned int, flags) -+{ -+#ifdef CONFIG_KSM -+ if (flags != 0) -+ return -EINVAL; -+ -+ return do_process_ksm_control(pidfd, PKSM_DISABLE); -+#else /* CONFIG_KSM */ -+ return -ENOSYS; -+#endif /* CONFIG_KSM */ -+} -+ -+SYSCALL_DEFINE2(process_ksm_status, int, pidfd, unsigned int, flags) -+{ -+#ifdef CONFIG_KSM -+ if (flags != 0) -+ return -EINVAL; -+ -+ return do_process_ksm_control(pidfd, PKSM_STATUS); -+#else /* CONFIG_KSM */ -+ return -ENOSYS; -+#endif /* CONFIG_KSM */ -+} -+ -+#ifdef CONFIG_KSM -+static ssize_t process_ksm_enable_show(struct kobject *kobj, -+ struct kobj_attribute *attr, char *buf) -+{ -+ return sprintf(buf, "%u\n", __NR_process_ksm_enable); -+} -+static struct kobj_attribute process_ksm_enable_attr = __ATTR_RO(process_ksm_enable); -+ -+static ssize_t process_ksm_disable_show(struct kobject *kobj, -+ struct kobj_attribute *attr, char *buf) -+{ -+ return sprintf(buf, "%u\n", __NR_process_ksm_disable); -+} -+static struct kobj_attribute process_ksm_disable_attr = __ATTR_RO(process_ksm_disable); -+ -+static ssize_t process_ksm_status_show(struct kobject *kobj, -+ struct kobj_attribute *attr, char *buf) -+{ -+ return sprintf(buf, "%u\n", __NR_process_ksm_status); -+} -+static struct kobj_attribute process_ksm_status_attr = __ATTR_RO(process_ksm_status); -+ -+static struct attribute *process_ksm_sysfs_attrs[] = { -+ &process_ksm_enable_attr.attr, -+ &process_ksm_disable_attr.attr, -+ &process_ksm_status_attr.attr, -+ NULL, -+}; -+ -+static const struct attribute_group process_ksm_sysfs_attr_group = { -+ .attrs = process_ksm_sysfs_attrs, -+ .name = "process_ksm", -+}; -+ -+static int __init process_ksm_sysfs_init(void) -+{ -+ return sysfs_create_group(kernel_kobj, &process_ksm_sysfs_attr_group); -+} -+subsys_initcall(process_ksm_sysfs_init); -+#endif /* CONFIG_KSM */ -+ - SYSCALL_DEFINE3(getcpu, unsigned __user *, cpup, unsigned __user *, nodep, - struct getcpu_cache __user *, unused) - { -diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c -index c00a86931f8c..d82213d68522 100644 ---- a/kernel/sys_ni.c -+++ b/kernel/sys_ni.c -@@ -186,6 +186,9 @@ COND_SYSCALL(mincore); - COND_SYSCALL(madvise); - COND_SYSCALL(process_madvise); - COND_SYSCALL(process_mrelease); -+COND_SYSCALL(process_ksm_enable); -+COND_SYSCALL(process_ksm_disable); -+COND_SYSCALL(process_ksm_status); - COND_SYSCALL(remap_file_pages); - COND_SYSCALL(mbind); - COND_SYSCALL(get_mempolicy); -diff --git a/scripts/syscall.tbl b/scripts/syscall.tbl -index ebbdb3c42e9f..b19b6bfe5cd4 100644 ---- a/scripts/syscall.tbl -+++ b/scripts/syscall.tbl -@@ -407,3 +407,6 @@ - 464 common getxattrat sys_getxattrat - 465 common listxattrat sys_listxattrat - 466 common removexattrat sys_removexattrat -+467 common process_ksm_enable sys_process_ksm_enable -+468 common process_ksm_disable sys_process_ksm_disable -+469 common process_ksm_status sys_process_ksm_status -diff --git a/tools/perf/arch/powerpc/entry/syscalls/syscall.tbl b/tools/perf/arch/powerpc/entry/syscalls/syscall.tbl -index d8b4ab78bef0..57aa958c1b97 100644 ---- a/tools/perf/arch/powerpc/entry/syscalls/syscall.tbl -+++ b/tools/perf/arch/powerpc/entry/syscalls/syscall.tbl -@@ -557,3 +557,6 @@ - 464 common getxattrat sys_getxattrat - 465 common listxattrat sys_listxattrat - 466 common removexattrat sys_removexattrat -+467 common process_ksm_enable sys_process_ksm_enable -+468 common process_ksm_disable sys_process_ksm_disable -+469 common process_ksm_status sys_process_ksm_status -diff --git a/tools/perf/arch/s390/entry/syscalls/syscall.tbl b/tools/perf/arch/s390/entry/syscalls/syscall.tbl -index e9115b4d8b63..2afc778f2d17 100644 ---- a/tools/perf/arch/s390/entry/syscalls/syscall.tbl -+++ b/tools/perf/arch/s390/entry/syscalls/syscall.tbl -@@ -469,3 +469,6 @@ - 464 common getxattrat sys_getxattrat sys_getxattrat - 465 common listxattrat sys_listxattrat sys_listxattrat - 466 common removexattrat sys_removexattrat sys_removexattrat -+467 common process_ksm_enable sys_process_ksm_enable sys_process_ksm_enable -+468 common process_ksm_disable sys_process_ksm_disable sys_process_ksm_disable -+469 common process_ksm_status sys_process_ksm_status sys_process_ksm_status --- -2.48.0.rc1 - -From 6a47e0ec10d0964d0447b57904e228d8b32b0de1 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Sun, 2 Feb 2025 13:55:14 +0100 -Subject: [PATCH 11/12] t2 - -Signed-off-by: Peter Jung ---- - .../ABI/testing/sysfs-driver-hid-appletb-kbd | 13 + - Documentation/core-api/printk-formats.rst | 32 + - MAINTAINERS | 6 + - drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 3 + - drivers/gpu/drm/drm_format_helper.c | 54 + - drivers/gpu/drm/i915/display/intel_ddi.c | 4 + - drivers/gpu/drm/i915/display/intel_fbdev.c | 6 +- - drivers/gpu/drm/i915/display/intel_quirks.c | 15 + - drivers/gpu/drm/i915/display/intel_quirks.h | 1 + - .../gpu/drm/tests/drm_format_helper_test.c | 81 ++ - drivers/gpu/drm/tiny/Kconfig | 12 + - drivers/gpu/drm/tiny/Makefile | 1 + - drivers/gpu/drm/tiny/appletbdrm.c | 624 +++++++++ - drivers/gpu/vga/vga_switcheroo.c | 7 +- - drivers/hid/Kconfig | 26 + - drivers/hid/Makefile | 2 + - drivers/hid/hid-appletb-bl.c | 207 +++ - drivers/hid/hid-appletb-kbd.c | 506 ++++++++ - drivers/hid/hid-multitouch.c | 60 +- - drivers/hid/hid-quirks.c | 8 +- - drivers/hwmon/applesmc.c | 1138 ++++++++++++----- - drivers/input/mouse/bcm5974.c | 138 ++ - .../broadcom/brcm80211/brcmfmac/pcie.c | 4 +- - drivers/pci/vgaarb.c | 1 + - drivers/platform/x86/apple-gmux.c | 18 + - drivers/staging/Kconfig | 2 + - drivers/staging/Makefile | 1 + - drivers/staging/apple-bce/Kconfig | 18 + - drivers/staging/apple-bce/Makefile | 28 + - drivers/staging/apple-bce/apple_bce.c | 445 +++++++ - drivers/staging/apple-bce/apple_bce.h | 38 + - drivers/staging/apple-bce/audio/audio.c | 711 ++++++++++ - drivers/staging/apple-bce/audio/audio.h | 125 ++ - drivers/staging/apple-bce/audio/description.h | 42 + - drivers/staging/apple-bce/audio/pcm.c | 308 +++++ - drivers/staging/apple-bce/audio/pcm.h | 16 + - drivers/staging/apple-bce/audio/protocol.c | 347 +++++ - drivers/staging/apple-bce/audio/protocol.h | 147 +++ - .../staging/apple-bce/audio/protocol_bce.c | 226 ++++ - .../staging/apple-bce/audio/protocol_bce.h | 72 ++ - drivers/staging/apple-bce/mailbox.c | 151 +++ - drivers/staging/apple-bce/mailbox.h | 53 + - drivers/staging/apple-bce/queue.c | 390 ++++++ - drivers/staging/apple-bce/queue.h | 177 +++ - drivers/staging/apple-bce/queue_dma.c | 220 ++++ - drivers/staging/apple-bce/queue_dma.h | 50 + - drivers/staging/apple-bce/vhci/command.h | 204 +++ - drivers/staging/apple-bce/vhci/queue.c | 268 ++++ - drivers/staging/apple-bce/vhci/queue.h | 76 ++ - drivers/staging/apple-bce/vhci/transfer.c | 661 ++++++++++ - drivers/staging/apple-bce/vhci/transfer.h | 73 ++ - drivers/staging/apple-bce/vhci/vhci.c | 759 +++++++++++ - drivers/staging/apple-bce/vhci/vhci.h | 52 + - include/drm/drm_format_helper.h | 3 + - lib/test_printf.c | 20 +- - lib/vsprintf.c | 36 +- - scripts/checkpatch.pl | 2 +- - 57 files changed, 8350 insertions(+), 338 deletions(-) - create mode 100644 Documentation/ABI/testing/sysfs-driver-hid-appletb-kbd - create mode 100644 drivers/gpu/drm/tiny/appletbdrm.c - create mode 100644 drivers/hid/hid-appletb-bl.c - create mode 100644 drivers/hid/hid-appletb-kbd.c - create mode 100644 drivers/staging/apple-bce/Kconfig - create mode 100644 drivers/staging/apple-bce/Makefile - create mode 100644 drivers/staging/apple-bce/apple_bce.c - create mode 100644 drivers/staging/apple-bce/apple_bce.h - create mode 100644 drivers/staging/apple-bce/audio/audio.c - create mode 100644 drivers/staging/apple-bce/audio/audio.h - create mode 100644 drivers/staging/apple-bce/audio/description.h - create mode 100644 drivers/staging/apple-bce/audio/pcm.c - create mode 100644 drivers/staging/apple-bce/audio/pcm.h - create mode 100644 drivers/staging/apple-bce/audio/protocol.c - create mode 100644 drivers/staging/apple-bce/audio/protocol.h - create mode 100644 drivers/staging/apple-bce/audio/protocol_bce.c - create mode 100644 drivers/staging/apple-bce/audio/protocol_bce.h - create mode 100644 drivers/staging/apple-bce/mailbox.c - create mode 100644 drivers/staging/apple-bce/mailbox.h - create mode 100644 drivers/staging/apple-bce/queue.c - create mode 100644 drivers/staging/apple-bce/queue.h - create mode 100644 drivers/staging/apple-bce/queue_dma.c - create mode 100644 drivers/staging/apple-bce/queue_dma.h - create mode 100644 drivers/staging/apple-bce/vhci/command.h - create mode 100644 drivers/staging/apple-bce/vhci/queue.c - create mode 100644 drivers/staging/apple-bce/vhci/queue.h - create mode 100644 drivers/staging/apple-bce/vhci/transfer.c - create mode 100644 drivers/staging/apple-bce/vhci/transfer.h - create mode 100644 drivers/staging/apple-bce/vhci/vhci.c - create mode 100644 drivers/staging/apple-bce/vhci/vhci.h - -diff --git a/Documentation/ABI/testing/sysfs-driver-hid-appletb-kbd b/Documentation/ABI/testing/sysfs-driver-hid-appletb-kbd -new file mode 100644 -index 000000000000..2a19584d091e ---- /dev/null -+++ b/Documentation/ABI/testing/sysfs-driver-hid-appletb-kbd -@@ -0,0 +1,13 @@ -+What: /sys/bus/hid/drivers/hid-appletb-kbd//mode -+Date: September, 2023 -+KernelVersion: 6.5 -+Contact: linux-input@vger.kernel.org -+Description: -+ The set of keys displayed on the Touch Bar. -+ Valid values are: -+ == ================= -+ 0 Escape key only -+ 1 Function keys -+ 2 Media/brightness keys -+ 3 None -+ == ================= -diff --git a/Documentation/core-api/printk-formats.rst b/Documentation/core-api/printk-formats.rst -index ecccc0473da9..6de6b0e6abf3 100644 ---- a/Documentation/core-api/printk-formats.rst -+++ b/Documentation/core-api/printk-formats.rst -@@ -648,6 +648,38 @@ Examples:: - %p4cc Y10 little-endian (0x20303159) - %p4cc NV12 big-endian (0xb231564e) - -+Generic FourCC code -+------------------- -+ -+:: -+ %p4c[hnbl] gP00 (0x67503030) -+ -+Print a generic FourCC code, as both ASCII characters and its numerical -+value as hexadecimal. -+ -+The additional ``h``, ``r``, ``b``, and ``l`` specifiers are used to specify -+host, reversed, big or little endian order data respectively. Host endian -+order means the data is interpreted as a 32-bit integer and the most -+significant byte is printed first; that is, the character code as printed -+matches the byte order stored in memory on big-endian systems, and is reversed -+on little-endian systems. -+ -+Passed by reference. -+ -+Examples for a little-endian machine, given &(u32)0x67503030:: -+ -+ %p4ch gP00 (0x67503030) -+ %p4cl gP00 (0x67503030) -+ %p4cb 00Pg (0x30305067) -+ %p4cr 00Pg (0x30305067) -+ -+Examples for a big-endian machine, given &(u32)0x67503030:: -+ -+ %p4ch gP00 (0x67503030) -+ %p4cl 00Pg (0x30305067) -+ %p4cb gP00 (0x67503030) -+ %p4cr 00Pg (0x30305067) -+ - Rust - ---- - -diff --git a/MAINTAINERS b/MAINTAINERS -index efecb59adfe6..16af42c68cca 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -7066,6 +7066,12 @@ S: Supported - T: git https://gitlab.freedesktop.org/drm/misc/kernel.git - F: drivers/gpu/drm/sun4i/sun8i* - -+DRM DRIVER FOR APPLE TOUCH BARS -+M: Kerem Karabay -+L: dri-devel@lists.freedesktop.org -+S: Maintained -+F: drivers/gpu/drm/tiny/appletbdrm.c -+ - DRM DRIVER FOR ARM PL111 CLCD - S: Orphan - T: git https://gitlab.freedesktop.org/drm/misc/kernel.git -diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c -index 811d020f3f4b..ade5a5b597e4 100644 ---- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c -+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c -@@ -2256,6 +2256,9 @@ static int amdgpu_pci_probe(struct pci_dev *pdev, - int ret, retry = 0, i; - bool supports_atomic = false; - -+ if (vga_switcheroo_client_probe_defer(pdev)) -+ return -EPROBE_DEFER; -+ - /* skip devices which are owned by radeon */ - for (i = 0; i < ARRAY_SIZE(amdgpu_unsupported_pciidlist); i++) { - if (amdgpu_unsupported_pciidlist[i] == pdev->device) -diff --git a/drivers/gpu/drm/drm_format_helper.c b/drivers/gpu/drm/drm_format_helper.c -index b1be458ed4dd..28c0e76a1e88 100644 ---- a/drivers/gpu/drm/drm_format_helper.c -+++ b/drivers/gpu/drm/drm_format_helper.c -@@ -702,6 +702,57 @@ void drm_fb_xrgb8888_to_rgb888(struct iosys_map *dst, const unsigned int *dst_pi - } - EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb888); - -+static void drm_fb_xrgb8888_to_bgr888_line(void *dbuf, const void *sbuf, unsigned int pixels) -+{ -+ u8 *dbuf8 = dbuf; -+ const __le32 *sbuf32 = sbuf; -+ unsigned int x; -+ u32 pix; -+ -+ for (x = 0; x < pixels; x++) { -+ pix = le32_to_cpu(sbuf32[x]); -+ /* write red-green-blue to output in little endianness */ -+ *dbuf8++ = (pix & 0x00FF0000) >> 16; -+ *dbuf8++ = (pix & 0x0000FF00) >> 8; -+ *dbuf8++ = (pix & 0x000000FF) >> 0; -+ } -+} -+ -+/** -+ * drm_fb_xrgb8888_to_bgr888 - Convert XRGB8888 to BGR888 clip buffer -+ * @dst: Array of BGR888 destination buffers -+ * @dst_pitch: Array of numbers of bytes between the start of two consecutive scanlines -+ * within @dst; can be NULL if scanlines are stored next to each other. -+ * @src: Array of XRGB8888 source buffers -+ * @fb: DRM framebuffer -+ * @clip: Clip rectangle area to copy -+ * @state: Transform and conversion state -+ * -+ * This function copies parts of a framebuffer to display memory and converts the -+ * color format during the process. Destination and framebuffer formats must match. The -+ * parameters @dst, @dst_pitch and @src refer to arrays. Each array must have at -+ * least as many entries as there are planes in @fb's format. Each entry stores the -+ * value for the format's respective color plane at the same index. -+ * -+ * This function does not apply clipping on @dst (i.e. the destination is at the -+ * top-left corner). -+ * -+ * Drivers can use this function for BGR888 devices that don't natively -+ * support XRGB8888. -+ */ -+void drm_fb_xrgb8888_to_bgr888(struct iosys_map *dst, const unsigned int *dst_pitch, -+ const struct iosys_map *src, const struct drm_framebuffer *fb, -+ const struct drm_rect *clip, struct drm_format_conv_state *state) -+{ -+ static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = { -+ 3, -+ }; -+ -+ drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state, -+ drm_fb_xrgb8888_to_bgr888_line); -+} -+EXPORT_SYMBOL(drm_fb_xrgb8888_to_bgr888); -+ - static void drm_fb_xrgb8888_to_argb8888_line(void *dbuf, const void *sbuf, unsigned int pixels) - { - __le32 *dbuf32 = dbuf; -@@ -1035,6 +1086,9 @@ int drm_fb_blit(struct iosys_map *dst, const unsigned int *dst_pitch, uint32_t d - } else if (dst_format == DRM_FORMAT_RGB888) { - drm_fb_xrgb8888_to_rgb888(dst, dst_pitch, src, fb, clip, state); - return 0; -+ } else if (dst_format == DRM_FORMAT_BGR888) { -+ drm_fb_xrgb8888_to_bgr888(dst, dst_pitch, src, fb, clip, state); -+ return 0; - } else if (dst_format == DRM_FORMAT_ARGB8888) { - drm_fb_xrgb8888_to_argb8888(dst, dst_pitch, src, fb, clip, state); - return 0; -diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c -index 49b5cc01ce40..1435f49f2ce6 100644 ---- a/drivers/gpu/drm/i915/display/intel_ddi.c -+++ b/drivers/gpu/drm/i915/display/intel_ddi.c -@@ -4685,6 +4685,7 @@ intel_ddi_init_hdmi_connector(struct intel_digital_port *dig_port) - - static bool intel_ddi_a_force_4_lanes(struct intel_digital_port *dig_port) - { -+ struct intel_display *display = to_intel_display(dig_port); - struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); - - if (dig_port->base.port != PORT_A) -@@ -4693,6 +4694,9 @@ static bool intel_ddi_a_force_4_lanes(struct intel_digital_port *dig_port) - if (dig_port->saved_port_bits & DDI_A_4_LANES) - return false; - -+ if (intel_has_quirk(display, QUIRK_DDI_A_FORCE_4_LANES)) -+ return true; -+ - /* Broxton/Geminilake: Bspec says that DDI_A_4_LANES is the only - * supported configuration - */ -diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c -index 00852ff5b247..4c56f1b622be 100644 ---- a/drivers/gpu/drm/i915/display/intel_fbdev.c -+++ b/drivers/gpu/drm/i915/display/intel_fbdev.c -@@ -197,10 +197,10 @@ static int intelfb_create(struct drm_fb_helper *helper, - ifbdev->fb = NULL; - - if (fb && -- (sizes->fb_width > fb->base.width || -- sizes->fb_height > fb->base.height)) { -+ (sizes->fb_width != fb->base.width || -+ sizes->fb_height != fb->base.height)) { - drm_dbg_kms(&dev_priv->drm, -- "BIOS fb too small (%dx%d), we require (%dx%d)," -+ "BIOS fb not valid (%dx%d), we require (%dx%d)," - " releasing it\n", - fb->base.width, fb->base.height, - sizes->fb_width, sizes->fb_height); -diff --git a/drivers/gpu/drm/i915/display/intel_quirks.c b/drivers/gpu/drm/i915/display/intel_quirks.c -index 28f497ae785b..c2952b0f8b88 100644 ---- a/drivers/gpu/drm/i915/display/intel_quirks.c -+++ b/drivers/gpu/drm/i915/display/intel_quirks.c -@@ -64,6 +64,18 @@ static void quirk_increase_ddi_disabled_time(struct intel_display *display) - drm_info(display->drm, "Applying Increase DDI Disabled quirk\n"); - } - -+/* -+ * In some cases, the firmware might not set the lane count to 4 (for example, -+ * when booting in some dual GPU Macs with the dGPU as the default GPU), this -+ * quirk is used to force it as otherwise it might not be possible to compute a -+ * valid link configuration. -+ */ -+static void quirk_ddi_a_force_4_lanes(struct intel_display *display) -+{ -+ intel_set_quirk(display, QUIRK_DDI_A_FORCE_4_LANES); -+ drm_info(display->drm, "Applying DDI A Forced 4 Lanes quirk\n"); -+} -+ - static void quirk_no_pps_backlight_power_hook(struct intel_display *display) - { - intel_set_quirk(display, QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK); -@@ -229,6 +241,9 @@ static struct intel_quirk intel_quirks[] = { - { 0x3184, 0x1019, 0xa94d, quirk_increase_ddi_disabled_time }, - /* HP Notebook - 14-r206nv */ - { 0x0f31, 0x103c, 0x220f, quirk_invert_brightness }, -+ -+ /* Apple MacBookPro15,1 */ -+ { 0x3e9b, 0x106b, 0x0176, quirk_ddi_a_force_4_lanes }, - }; - - static const struct intel_dpcd_quirk intel_dpcd_quirks[] = { -diff --git a/drivers/gpu/drm/i915/display/intel_quirks.h b/drivers/gpu/drm/i915/display/intel_quirks.h -index cafdebda7535..a5296f82776e 100644 ---- a/drivers/gpu/drm/i915/display/intel_quirks.h -+++ b/drivers/gpu/drm/i915/display/intel_quirks.h -@@ -20,6 +20,7 @@ enum intel_quirk_id { - QUIRK_LVDS_SSC_DISABLE, - QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK, - QUIRK_FW_SYNC_LEN, -+ QUIRK_DDI_A_FORCE_4_LANES, - }; - - void intel_init_quirks(struct intel_display *display); -diff --git a/drivers/gpu/drm/tests/drm_format_helper_test.c b/drivers/gpu/drm/tests/drm_format_helper_test.c -index 08992636ec05..35cd3405d045 100644 ---- a/drivers/gpu/drm/tests/drm_format_helper_test.c -+++ b/drivers/gpu/drm/tests/drm_format_helper_test.c -@@ -60,6 +60,11 @@ struct convert_to_rgb888_result { - const u8 expected[TEST_BUF_SIZE]; - }; - -+struct convert_to_bgr888_result { -+ unsigned int dst_pitch; -+ const u8 expected[TEST_BUF_SIZE]; -+}; -+ - struct convert_to_argb8888_result { - unsigned int dst_pitch; - const u32 expected[TEST_BUF_SIZE]; -@@ -107,6 +112,7 @@ struct convert_xrgb8888_case { - struct convert_to_argb1555_result argb1555_result; - struct convert_to_rgba5551_result rgba5551_result; - struct convert_to_rgb888_result rgb888_result; -+ struct convert_to_bgr888_result bgr888_result; - struct convert_to_argb8888_result argb8888_result; - struct convert_to_xrgb2101010_result xrgb2101010_result; - struct convert_to_argb2101010_result argb2101010_result; -@@ -151,6 +157,10 @@ static struct convert_xrgb8888_case convert_xrgb8888_cases[] = { - .dst_pitch = TEST_USE_DEFAULT_PITCH, - .expected = { 0x00, 0x00, 0xFF }, - }, -+ .bgr888_result = { -+ .dst_pitch = TEST_USE_DEFAULT_PITCH, -+ .expected = { 0xFF, 0x00, 0x00 }, -+ }, - .argb8888_result = { - .dst_pitch = TEST_USE_DEFAULT_PITCH, - .expected = { 0xFFFF0000 }, -@@ -217,6 +227,10 @@ static struct convert_xrgb8888_case convert_xrgb8888_cases[] = { - .dst_pitch = TEST_USE_DEFAULT_PITCH, - .expected = { 0x00, 0x00, 0xFF }, - }, -+ .bgr888_result = { -+ .dst_pitch = TEST_USE_DEFAULT_PITCH, -+ .expected = { 0xFF, 0x00, 0x00 }, -+ }, - .argb8888_result = { - .dst_pitch = TEST_USE_DEFAULT_PITCH, - .expected = { 0xFFFF0000 }, -@@ -330,6 +344,15 @@ static struct convert_xrgb8888_case convert_xrgb8888_cases[] = { - 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, - }, - }, -+ .bgr888_result = { -+ .dst_pitch = TEST_USE_DEFAULT_PITCH, -+ .expected = { -+ 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, -+ 0xFF, 0x00, 0x00, 0x00, 0xFF, 0x00, -+ 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, -+ 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, -+ }, -+ }, - .argb8888_result = { - .dst_pitch = TEST_USE_DEFAULT_PITCH, - .expected = { -@@ -468,6 +491,17 @@ static struct convert_xrgb8888_case convert_xrgb8888_cases[] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - }, - }, -+ .bgr888_result = { -+ .dst_pitch = 15, -+ .expected = { -+ 0x0E, 0x44, 0x9C, 0x11, 0x4D, 0x05, 0xA8, 0xF3, 0x03, -+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -+ 0x6C, 0xF0, 0x73, 0x0E, 0x44, 0x9C, 0x11, 0x4D, 0x05, -+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -+ 0xA8, 0x03, 0x03, 0x6C, 0xF0, 0x73, 0x0E, 0x44, 0x9C, -+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -+ }, -+ }, - .argb8888_result = { - .dst_pitch = 20, - .expected = { -@@ -914,6 +948,52 @@ static void drm_test_fb_xrgb8888_to_rgb888(struct kunit *test) - KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size); - } - -+static void drm_test_fb_xrgb8888_to_bgr888(struct kunit *test) -+{ -+ const struct convert_xrgb8888_case *params = test->param_value; -+ const struct convert_to_bgr888_result *result = ¶ms->bgr888_result; -+ size_t dst_size; -+ u8 *buf = NULL; -+ __le32 *xrgb8888 = NULL; -+ struct iosys_map dst, src; -+ -+ struct drm_framebuffer fb = { -+ .format = drm_format_info(DRM_FORMAT_XRGB8888), -+ .pitches = { params->pitch, 0, 0 }, -+ }; -+ -+ dst_size = conversion_buf_size(DRM_FORMAT_BGR888, result->dst_pitch, -+ ¶ms->clip, 0); -+ KUNIT_ASSERT_GT(test, dst_size, 0); -+ -+ buf = kunit_kzalloc(test, dst_size, GFP_KERNEL); -+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buf); -+ iosys_map_set_vaddr(&dst, buf); -+ -+ xrgb8888 = cpubuf_to_le32(test, params->xrgb8888, TEST_BUF_SIZE); -+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xrgb8888); -+ iosys_map_set_vaddr(&src, xrgb8888); -+ -+ /* -+ * BGR888 expected results are already in little-endian -+ * order, so there's no need to convert the test output. -+ */ -+ drm_fb_xrgb8888_to_bgr888(&dst, &result->dst_pitch, &src, &fb, ¶ms->clip, -+ &fmtcnv_state); -+ KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size); -+ -+ buf = dst.vaddr; /* restore original value of buf */ -+ memset(buf, 0, dst_size); -+ -+ int blit_result = 0; -+ -+ blit_result = drm_fb_blit(&dst, &result->dst_pitch, DRM_FORMAT_BGR888, &src, &fb, ¶ms->clip, -+ &fmtcnv_state); -+ -+ KUNIT_EXPECT_FALSE(test, blit_result); -+ KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size); -+} -+ - static void drm_test_fb_xrgb8888_to_argb8888(struct kunit *test) - { - const struct convert_xrgb8888_case *params = test->param_value; -@@ -1851,6 +1931,7 @@ static struct kunit_case drm_format_helper_test_cases[] = { - KUNIT_CASE_PARAM(drm_test_fb_xrgb8888_to_argb1555, convert_xrgb8888_gen_params), - KUNIT_CASE_PARAM(drm_test_fb_xrgb8888_to_rgba5551, convert_xrgb8888_gen_params), - KUNIT_CASE_PARAM(drm_test_fb_xrgb8888_to_rgb888, convert_xrgb8888_gen_params), -+ KUNIT_CASE_PARAM(drm_test_fb_xrgb8888_to_bgr888, convert_xrgb8888_gen_params), - KUNIT_CASE_PARAM(drm_test_fb_xrgb8888_to_argb8888, convert_xrgb8888_gen_params), - KUNIT_CASE_PARAM(drm_test_fb_xrgb8888_to_xrgb2101010, convert_xrgb8888_gen_params), - KUNIT_CASE_PARAM(drm_test_fb_xrgb8888_to_argb2101010, convert_xrgb8888_gen_params), -diff --git a/drivers/gpu/drm/tiny/Kconfig b/drivers/gpu/drm/tiny/Kconfig -index 94cbdb1337c0..1201aee7bab3 100644 ---- a/drivers/gpu/drm/tiny/Kconfig -+++ b/drivers/gpu/drm/tiny/Kconfig -@@ -1,5 +1,17 @@ - # SPDX-License-Identifier: GPL-2.0-only - -+config DRM_APPLETBDRM -+ tristate "DRM support for Apple Touch Bars" -+ depends on DRM && USB && MMU -+ select DRM_KMS_HELPER -+ select DRM_GEM_SHMEM_HELPER -+ help -+ Say Y here if you want support for the display of Touch Bars on x86 -+ MacBook Pros. -+ -+ To compile this driver as a module, choose M here: the -+ module will be called appletbdrm. -+ - config DRM_ARCPGU - tristate "ARC PGU" - depends on DRM && OF -diff --git a/drivers/gpu/drm/tiny/Makefile b/drivers/gpu/drm/tiny/Makefile -index 4aaf56f8707d..d9add9c3eda3 100644 ---- a/drivers/gpu/drm/tiny/Makefile -+++ b/drivers/gpu/drm/tiny/Makefile -@@ -1,5 +1,6 @@ - # SPDX-License-Identifier: GPL-2.0-only - -+obj-$(CONFIG_DRM_APPLETBDRM) += appletbdrm.o - obj-$(CONFIG_DRM_ARCPGU) += arcpgu.o - obj-$(CONFIG_DRM_BOCHS) += bochs.o - obj-$(CONFIG_DRM_CIRRUS_QEMU) += cirrus.o -diff --git a/drivers/gpu/drm/tiny/appletbdrm.c b/drivers/gpu/drm/tiny/appletbdrm.c -new file mode 100644 -index 000000000000..7a74c8ad37cd ---- /dev/null -+++ b/drivers/gpu/drm/tiny/appletbdrm.c -@@ -0,0 +1,624 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* -+ * Apple Touch Bar DRM Driver -+ * -+ * Copyright (c) 2023 Kerem Karabay -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+ -+#include -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#define _APPLETBDRM_FOURCC(s) (((s)[0] << 24) | ((s)[1] << 16) | ((s)[2] << 8) | (s)[3]) -+#define APPLETBDRM_FOURCC(s) _APPLETBDRM_FOURCC(#s) -+ -+#define APPLETBDRM_PIXEL_FORMAT APPLETBDRM_FOURCC(RGBA) /* The actual format is BGR888 */ -+#define APPLETBDRM_BITS_PER_PIXEL 24 -+ -+#define APPLETBDRM_MSG_CLEAR_DISPLAY APPLETBDRM_FOURCC(CLRD) -+#define APPLETBDRM_MSG_GET_INFORMATION APPLETBDRM_FOURCC(GINF) -+#define APPLETBDRM_MSG_UPDATE_COMPLETE APPLETBDRM_FOURCC(UDCL) -+#define APPLETBDRM_MSG_SIGNAL_READINESS APPLETBDRM_FOURCC(REDY) -+ -+#define APPLETBDRM_BULK_MSG_TIMEOUT 1000 -+ -+#define drm_to_adev(_drm) container_of(_drm, struct appletbdrm_device, drm) -+#define adev_to_udev(adev) interface_to_usbdev(to_usb_interface(adev->dev)) -+ -+struct appletbdrm_device { -+ struct device *dev; -+ -+ u8 in_ep; -+ u8 out_ep; -+ -+ u32 width; -+ u32 height; -+ -+ struct drm_device drm; -+ struct drm_display_mode mode; -+ struct drm_connector connector; -+ struct drm_simple_display_pipe pipe; -+ -+ bool readiness_signal_received; -+}; -+ -+struct appletbdrm_request_header { -+ __le16 unk_00; -+ __le16 unk_02; -+ __le32 unk_04; -+ __le32 unk_08; -+ __le32 size; -+} __packed; -+ -+struct appletbdrm_response_header { -+ u8 unk_00[16]; -+ u32 msg; -+} __packed; -+ -+struct appletbdrm_simple_request { -+ struct appletbdrm_request_header header; -+ u32 msg; -+ u8 unk_14[8]; -+ __le32 size; -+} __packed; -+ -+struct appletbdrm_information { -+ struct appletbdrm_response_header header; -+ u8 unk_14[12]; -+ __le32 width; -+ __le32 height; -+ u8 bits_per_pixel; -+ __le32 bytes_per_row; -+ __le32 orientation; -+ __le32 bitmap_info; -+ u32 pixel_format; -+ __le32 width_inches; /* floating point */ -+ __le32 height_inches; /* floating point */ -+} __packed; -+ -+struct appletbdrm_frame { -+ __le16 begin_x; -+ __le16 begin_y; -+ __le16 width; -+ __le16 height; -+ __le32 buf_size; -+ u8 buf[]; -+} __packed; -+ -+struct appletbdrm_fb_request_footer { -+ u8 unk_00[12]; -+ __le32 unk_0c; -+ u8 unk_10[12]; -+ __le32 unk_1c; -+ __le64 timestamp; -+ u8 unk_28[12]; -+ __le32 unk_34; -+ u8 unk_38[20]; -+ __le32 unk_4c; -+} __packed; -+ -+struct appletbdrm_fb_request { -+ struct appletbdrm_request_header header; -+ __le16 unk_10; -+ u8 msg_id; -+ u8 unk_13[29]; -+ /* -+ * Contents of `data`: -+ * - struct appletbdrm_frame frames[]; -+ * - struct appletbdrm_fb_request_footer footer; -+ * - padding to make the total size a multiple of 16 -+ */ -+ u8 data[]; -+} __packed; -+ -+struct appletbdrm_fb_request_response { -+ struct appletbdrm_response_header header; -+ u8 unk_14[12]; -+ __le64 timestamp; -+} __packed; -+ -+static int appletbdrm_send_request(struct appletbdrm_device *adev, -+ struct appletbdrm_request_header *request, size_t size) -+{ -+ struct usb_device *udev = adev_to_udev(adev); -+ struct drm_device *drm = &adev->drm; -+ int ret, actual_size; -+ -+ ret = usb_bulk_msg(udev, usb_sndbulkpipe(udev, adev->out_ep), -+ request, size, &actual_size, APPLETBDRM_BULK_MSG_TIMEOUT); -+ if (ret) { -+ drm_err(drm, "Failed to send message (%pe)\n", ERR_PTR(ret)); -+ return ret; -+ } -+ -+ if (actual_size != size) { -+ drm_err(drm, "Actual size (%d) doesn't match expected size (%lu)\n", -+ actual_size, size); -+ return -EIO; -+ } -+ -+ return ret; -+} -+ -+static int appletbdrm_read_response(struct appletbdrm_device *adev, -+ struct appletbdrm_response_header *response, -+ size_t size, u32 expected_response) -+{ -+ struct usb_device *udev = adev_to_udev(adev); -+ struct drm_device *drm = &adev->drm; -+ int ret, actual_size; -+ -+retry: -+ ret = usb_bulk_msg(udev, usb_rcvbulkpipe(udev, adev->in_ep), -+ response, size, &actual_size, APPLETBDRM_BULK_MSG_TIMEOUT); -+ if (ret) { -+ drm_err(drm, "Failed to read response (%pe)\n", ERR_PTR(ret)); -+ return ret; -+ } -+ -+ /* -+ * The device responds to the first request sent in a particular -+ * timeframe after the USB device configuration is set with a readiness -+ * signal, in which case the response should be read again -+ */ -+ if (response->msg == APPLETBDRM_MSG_SIGNAL_READINESS) { -+ if (!adev->readiness_signal_received) { -+ adev->readiness_signal_received = true; -+ goto retry; -+ } -+ -+ drm_err(drm, "Encountered unexpected readiness signal\n"); -+ return -EIO; -+ } -+ -+ if (actual_size != size) { -+ drm_err(drm, "Actual size (%d) doesn't match expected size (%lu)\n", -+ actual_size, size); -+ return -EIO; -+ } -+ -+ if (response->msg != expected_response) { -+ drm_err(drm, "Unexpected response from device (expected %p4ch found %p4ch)\n", -+ &expected_response, &response->msg); -+ return -EIO; -+ } -+ -+ return 0; -+} -+ -+static int appletbdrm_send_msg(struct appletbdrm_device *adev, u32 msg) -+{ -+ struct appletbdrm_simple_request *request; -+ int ret; -+ -+ request = kzalloc(sizeof(*request), GFP_KERNEL); -+ if (!request) -+ return -ENOMEM; -+ -+ request->header.unk_00 = cpu_to_le16(2); -+ request->header.unk_02 = cpu_to_le16(0x1512); -+ request->header.size = cpu_to_le32(sizeof(*request) - sizeof(request->header)); -+ request->msg = msg; -+ request->size = request->header.size; -+ -+ ret = appletbdrm_send_request(adev, &request->header, sizeof(*request)); -+ -+ kfree(request); -+ -+ return ret; -+} -+ -+static int appletbdrm_clear_display(struct appletbdrm_device *adev) -+{ -+ return appletbdrm_send_msg(adev, APPLETBDRM_MSG_CLEAR_DISPLAY); -+} -+ -+static int appletbdrm_signal_readiness(struct appletbdrm_device *adev) -+{ -+ return appletbdrm_send_msg(adev, APPLETBDRM_MSG_SIGNAL_READINESS); -+} -+ -+static int appletbdrm_get_information(struct appletbdrm_device *adev) -+{ -+ struct appletbdrm_information *info; -+ struct drm_device *drm = &adev->drm; -+ u8 bits_per_pixel; -+ u32 pixel_format; -+ int ret; -+ -+ info = kzalloc(sizeof(*info), GFP_KERNEL); -+ if (!info) -+ return -ENOMEM; -+ -+ ret = appletbdrm_send_msg(adev, APPLETBDRM_MSG_GET_INFORMATION); -+ if (ret) -+ return ret; -+ -+ ret = appletbdrm_read_response(adev, &info->header, sizeof(*info), -+ APPLETBDRM_MSG_GET_INFORMATION); -+ if (ret) -+ goto free_info; -+ -+ bits_per_pixel = info->bits_per_pixel; -+ pixel_format = get_unaligned(&info->pixel_format); -+ -+ adev->width = get_unaligned_le32(&info->width); -+ adev->height = get_unaligned_le32(&info->height); -+ -+ if (bits_per_pixel != APPLETBDRM_BITS_PER_PIXEL) { -+ drm_err(drm, "Encountered unexpected bits per pixel value (%d)\n", bits_per_pixel); -+ ret = -EINVAL; -+ goto free_info; -+ } -+ -+ if (pixel_format != APPLETBDRM_PIXEL_FORMAT) { -+ drm_err(drm, "Encountered unknown pixel format (%p4ch)\n", &pixel_format); -+ ret = -EINVAL; -+ goto free_info; -+ } -+ -+free_info: -+ kfree(info); -+ -+ return ret; -+} -+ -+static u32 rect_size(struct drm_rect *rect) -+{ -+ return drm_rect_width(rect) * drm_rect_height(rect) * (APPLETBDRM_BITS_PER_PIXEL / 8); -+} -+ -+static int appletbdrm_flush_damage(struct appletbdrm_device *adev, -+ struct drm_plane_state *old_state, -+ struct drm_plane_state *state) -+{ -+ struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(state); -+ struct appletbdrm_fb_request_response *response; -+ struct appletbdrm_fb_request_footer *footer; -+ struct drm_atomic_helper_damage_iter iter; -+ struct drm_framebuffer *fb = state->fb; -+ struct appletbdrm_fb_request *request; -+ struct drm_device *drm = &adev->drm; -+ struct appletbdrm_frame *frame; -+ u64 timestamp = ktime_get_ns(); -+ struct drm_rect damage; -+ size_t frames_size = 0; -+ size_t request_size; -+ int ret; -+ -+ drm_atomic_helper_damage_iter_init(&iter, old_state, state); -+ drm_atomic_for_each_plane_damage(&iter, &damage) { -+ frames_size += struct_size(frame, buf, rect_size(&damage)); -+ } -+ -+ if (!frames_size) -+ return 0; -+ -+ request_size = ALIGN(sizeof(*request) + frames_size + sizeof(*footer), 16); -+ -+ request = kzalloc(request_size, GFP_KERNEL); -+ if (!request) -+ return -ENOMEM; -+ -+ response = kzalloc(sizeof(*response), GFP_KERNEL); -+ if (!response) { -+ ret = -ENOMEM; -+ goto free_request; -+ } -+ -+ ret = drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE); -+ if (ret) { -+ drm_err(drm, "Failed to start CPU framebuffer access (%pe)\n", ERR_PTR(ret)); -+ goto free_response; -+ } -+ -+ request->header.unk_00 = cpu_to_le16(2); -+ request->header.unk_02 = cpu_to_le16(0x12); -+ request->header.unk_04 = cpu_to_le32(9); -+ request->header.size = cpu_to_le32(request_size - sizeof(request->header)); -+ request->unk_10 = cpu_to_le16(1); -+ request->msg_id = timestamp & 0xff; -+ -+ frame = (struct appletbdrm_frame *)request->data; -+ -+ drm_atomic_helper_damage_iter_init(&iter, old_state, state); -+ drm_atomic_for_each_plane_damage(&iter, &damage) { -+ struct iosys_map dst = IOSYS_MAP_INIT_VADDR(frame->buf); -+ u32 buf_size = rect_size(&damage); -+ -+ /* -+ * The coordinates need to be translated to the coordinate -+ * system the device expects, see the comment in -+ * appletbdrm_setup_mode_config -+ */ -+ frame->begin_x = cpu_to_le16(damage.y1); -+ frame->begin_y = cpu_to_le16(adev->height - damage.x2); -+ frame->width = cpu_to_le16(drm_rect_height(&damage)); -+ frame->height = cpu_to_le16(drm_rect_width(&damage)); -+ frame->buf_size = cpu_to_le32(buf_size); -+ -+ ret = drm_fb_blit(&dst, NULL, DRM_FORMAT_BGR888, -+ &shadow_plane_state->data[0], fb, &damage, &shadow_plane_state->fmtcnv_state); -+ if (ret) { -+ drm_err(drm, "Failed to copy damage clip (%pe)\n", ERR_PTR(ret)); -+ goto end_fb_cpu_access; -+ } -+ -+ frame = (void *)frame + struct_size(frame, buf, buf_size); -+ } -+ -+ footer = (struct appletbdrm_fb_request_footer *)&request->data[frames_size]; -+ -+ footer->unk_0c = cpu_to_le32(0xfffe); -+ footer->unk_1c = cpu_to_le32(0x80001); -+ footer->unk_34 = cpu_to_le32(0x80002); -+ footer->unk_4c = cpu_to_le32(0xffff); -+ footer->timestamp = cpu_to_le64(timestamp); -+ -+ ret = appletbdrm_send_request(adev, &request->header, request_size); -+ if (ret) -+ goto end_fb_cpu_access; -+ -+ ret = appletbdrm_read_response(adev, &response->header, sizeof(*response), -+ APPLETBDRM_MSG_UPDATE_COMPLETE); -+ if (ret) -+ goto end_fb_cpu_access; -+ -+ if (response->timestamp != footer->timestamp) { -+ drm_err(drm, "Response timestamp (%llu) doesn't match request timestamp (%llu)\n", -+ le64_to_cpu(response->timestamp), timestamp); -+ goto end_fb_cpu_access; -+ } -+ -+end_fb_cpu_access: -+ drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE); -+free_response: -+ kfree(response); -+free_request: -+ kfree(request); -+ -+ return ret; -+} -+ -+static int appletbdrm_connector_helper_get_modes(struct drm_connector *connector) -+{ -+ struct appletbdrm_device *adev = drm_to_adev(connector->dev); -+ -+ return drm_connector_helper_get_modes_fixed(connector, &adev->mode); -+} -+ -+static enum drm_mode_status appletbdrm_pipe_mode_valid(struct drm_simple_display_pipe *pipe, -+ const struct drm_display_mode *mode) -+{ -+ struct drm_crtc *crtc = &pipe->crtc; -+ struct appletbdrm_device *adev = drm_to_adev(crtc->dev); -+ -+ return drm_crtc_helper_mode_valid_fixed(crtc, mode, &adev->mode); -+} -+ -+static void appletbdrm_pipe_disable(struct drm_simple_display_pipe *pipe) -+{ -+ struct appletbdrm_device *adev = drm_to_adev(pipe->crtc.dev); -+ int idx; -+ -+ if (!drm_dev_enter(&adev->drm, &idx)) -+ return; -+ -+ appletbdrm_clear_display(adev); -+ -+ drm_dev_exit(idx); -+} -+ -+static void appletbdrm_pipe_update(struct drm_simple_display_pipe *pipe, -+ struct drm_plane_state *old_state) -+{ -+ struct drm_crtc *crtc = &pipe->crtc; -+ struct appletbdrm_device *adev = drm_to_adev(crtc->dev); -+ int idx; -+ -+ if (!crtc->state->active || !drm_dev_enter(&adev->drm, &idx)) -+ return; -+ -+ appletbdrm_flush_damage(adev, old_state, pipe->plane.state); -+ -+ drm_dev_exit(idx); -+} -+ -+static const u32 appletbdrm_formats[] = { -+ DRM_FORMAT_BGR888, -+ DRM_FORMAT_XRGB8888, /* emulated */ -+}; -+ -+static const struct drm_mode_config_funcs appletbdrm_mode_config_funcs = { -+ .fb_create = drm_gem_fb_create_with_dirty, -+ .atomic_check = drm_atomic_helper_check, -+ .atomic_commit = drm_atomic_helper_commit, -+}; -+ -+static const struct drm_connector_funcs appletbdrm_connector_funcs = { -+ .reset = drm_atomic_helper_connector_reset, -+ .destroy = drm_connector_cleanup, -+ .fill_modes = drm_helper_probe_single_connector_modes, -+ .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, -+ .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, -+}; -+ -+static const struct drm_connector_helper_funcs appletbdrm_connector_helper_funcs = { -+ .get_modes = appletbdrm_connector_helper_get_modes, -+}; -+ -+static const struct drm_simple_display_pipe_funcs appletbdrm_pipe_funcs = { -+ DRM_GEM_SIMPLE_DISPLAY_PIPE_SHADOW_PLANE_FUNCS, -+ .update = appletbdrm_pipe_update, -+ .disable = appletbdrm_pipe_disable, -+ .mode_valid = appletbdrm_pipe_mode_valid, -+}; -+ -+DEFINE_DRM_GEM_FOPS(appletbdrm_drm_fops); -+ -+static const struct drm_driver appletbdrm_drm_driver = { -+ DRM_GEM_SHMEM_DRIVER_OPS, -+ .name = "appletbdrm", -+ .desc = "Apple Touch Bar DRM Driver", -+ .date = "20230910", -+ .major = 1, -+ .minor = 0, -+ .driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC, -+ .fops = &appletbdrm_drm_fops, -+}; -+ -+static int appletbdrm_setup_mode_config(struct appletbdrm_device *adev) -+{ -+ struct drm_connector *connector = &adev->connector; -+ struct drm_device *drm = &adev->drm; -+ struct device *dev = adev->dev; -+ int ret; -+ -+ ret = drmm_mode_config_init(drm); -+ if (ret) -+ return dev_err_probe(dev, ret, "Failed to initialize mode configuration\n"); -+ -+ /* -+ * The coordinate system used by the device is different from the -+ * coordinate system of the framebuffer in that the x and y axes are -+ * swapped, and that the y axis is inverted; so what the device reports -+ * as the height is actually the width of the framebuffer and vice -+ * versa -+ */ -+ drm->mode_config.min_width = 0; -+ drm->mode_config.min_height = 0; -+ drm->mode_config.max_width = max(adev->height, DRM_SHADOW_PLANE_MAX_WIDTH); -+ drm->mode_config.max_height = max(adev->width, DRM_SHADOW_PLANE_MAX_HEIGHT); -+ drm->mode_config.preferred_depth = APPLETBDRM_BITS_PER_PIXEL; -+ drm->mode_config.funcs = &appletbdrm_mode_config_funcs; -+ -+ adev->mode = (struct drm_display_mode) { -+ DRM_MODE_INIT(60, adev->height, adev->width, -+ DRM_MODE_RES_MM(adev->height, 218), -+ DRM_MODE_RES_MM(adev->width, 218)) -+ }; -+ -+ ret = drm_connector_init(drm, connector, -+ &appletbdrm_connector_funcs, DRM_MODE_CONNECTOR_USB); -+ if (ret) -+ return dev_err_probe(dev, ret, "Failed to initialize connector\n"); -+ -+ drm_connector_helper_add(connector, &appletbdrm_connector_helper_funcs); -+ -+ ret = drm_connector_set_panel_orientation(connector, -+ DRM_MODE_PANEL_ORIENTATION_RIGHT_UP); -+ if (ret) -+ return dev_err_probe(dev, ret, "Failed to set panel orientation\n"); -+ -+ connector->display_info.non_desktop = true; -+ ret = drm_object_property_set_value(&connector->base, -+ drm->mode_config.non_desktop_property, true); -+ if (ret) -+ return dev_err_probe(dev, ret, "Failed to set non-desktop property\n"); -+ -+ ret = drm_simple_display_pipe_init(drm, &adev->pipe, &appletbdrm_pipe_funcs, -+ appletbdrm_formats, ARRAY_SIZE(appletbdrm_formats), -+ NULL, &adev->connector); -+ if (ret) -+ return dev_err_probe(dev, ret, "Failed to initialize simple display pipe\n"); -+ -+ drm_plane_enable_fb_damage_clips(&adev->pipe.plane); -+ -+ drm_mode_config_reset(drm); -+ -+ ret = drm_dev_register(drm, 0); -+ if (ret) -+ return dev_err_probe(dev, ret, "Failed to register DRM device\n"); -+ -+ return 0; -+} -+ -+static int appletbdrm_probe(struct usb_interface *intf, -+ const struct usb_device_id *id) -+{ -+ struct usb_endpoint_descriptor *bulk_in, *bulk_out; -+ struct device *dev = &intf->dev; -+ struct appletbdrm_device *adev; -+ int ret; -+ -+ ret = usb_find_common_endpoints(intf->cur_altsetting, &bulk_in, &bulk_out, NULL, NULL); -+ if (ret) -+ return dev_err_probe(dev, ret, "Failed to find bulk endpoints\n"); -+ -+ adev = devm_drm_dev_alloc(dev, &appletbdrm_drm_driver, struct appletbdrm_device, drm); -+ if (IS_ERR(adev)) -+ return PTR_ERR(adev); -+ -+ adev->dev = dev; -+ adev->in_ep = bulk_in->bEndpointAddress; -+ adev->out_ep = bulk_out->bEndpointAddress; -+ -+ usb_set_intfdata(intf, adev); -+ -+ ret = appletbdrm_get_information(adev); -+ if (ret) -+ return dev_err_probe(dev, ret, "Failed to get display information\n"); -+ -+ ret = appletbdrm_signal_readiness(adev); -+ if (ret) -+ return dev_err_probe(dev, ret, "Failed to signal readiness\n"); -+ -+ ret = appletbdrm_clear_display(adev); -+ if (ret) -+ return dev_err_probe(dev, ret, "Failed to clear display\n"); -+ -+ return appletbdrm_setup_mode_config(adev); -+} -+ -+static void appletbdrm_disconnect(struct usb_interface *intf) -+{ -+ struct appletbdrm_device *adev = usb_get_intfdata(intf); -+ struct drm_device *drm = &adev->drm; -+ -+ drm_dev_unplug(drm); -+ drm_atomic_helper_shutdown(drm); -+} -+ -+static void appletbdrm_shutdown(struct usb_interface *intf) -+{ -+ struct appletbdrm_device *adev = usb_get_intfdata(intf); -+ -+ /* -+ * The framebuffer needs to be cleared on shutdown since its content -+ * persists across boots -+ */ -+ drm_atomic_helper_shutdown(&adev->drm); -+} -+ -+static const struct usb_device_id appletbdrm_usb_id_table[] = { -+ { USB_DEVICE_INTERFACE_CLASS(0x05ac, 0x8302, USB_CLASS_AUDIO_VIDEO) }, -+ {} -+}; -+MODULE_DEVICE_TABLE(usb, appletbdrm_usb_id_table); -+ -+static struct usb_driver appletbdrm_usb_driver = { -+ .name = "appletbdrm", -+ .probe = appletbdrm_probe, -+ .disconnect = appletbdrm_disconnect, -+ .shutdown = appletbdrm_shutdown, -+ .id_table = appletbdrm_usb_id_table, -+}; -+module_usb_driver(appletbdrm_usb_driver); -+ -+MODULE_AUTHOR("Kerem Karabay "); -+MODULE_DESCRIPTION("Apple Touch Bar DRM Driver"); -+MODULE_LICENSE("GPL"); -diff --git a/drivers/gpu/vga/vga_switcheroo.c b/drivers/gpu/vga/vga_switcheroo.c -index 18f2c92beff8..3de1bca45ed2 100644 ---- a/drivers/gpu/vga/vga_switcheroo.c -+++ b/drivers/gpu/vga/vga_switcheroo.c -@@ -438,12 +438,7 @@ find_active_client(struct list_head *head) - bool vga_switcheroo_client_probe_defer(struct pci_dev *pdev) - { - if ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY) { -- /* -- * apple-gmux is needed on pre-retina MacBook Pro -- * to probe the panel if pdev is the inactive GPU. -- */ -- if (apple_gmux_present() && pdev != vga_default_device() && -- !vgasr_priv.handler_flags) -+ if (apple_gmux_present() && !vgasr_priv.handler_flags) - return true; - } - -diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig -index 4d2a89d65b65..311f0ab5d42f 100644 ---- a/drivers/hid/Kconfig -+++ b/drivers/hid/Kconfig -@@ -148,6 +148,31 @@ config HID_APPLEIR - - Say Y here if you want support for Apple infrared remote control. - -+config HID_APPLETB_BL -+ tristate "Apple Touch Bar Backlight" -+ depends on BACKLIGHT_CLASS_DEVICE -+ help -+ Say Y here if you want support for the backlight of Touch Bars on x86 -+ MacBook Pros. -+ -+ To compile this driver as a module, choose M here: the -+ module will be called hid-appletb-bl. -+ -+config HID_APPLETB_KBD -+ tristate "Apple Touch Bar Keyboard Mode" -+ depends on USB_HID -+ depends on BACKLIGHT_CLASS_DEVICE -+ depends on INPUT -+ select INPUT_SPARSEKMAP -+ select HID_APPLETB_BL -+ help -+ Say Y here if you want support for the keyboard mode (escape, -+ function, media and brightness keys) of Touch Bars on x86 MacBook -+ Pros. -+ -+ To compile this driver as a module, choose M here: the -+ module will be called hid-appletb-kbd. -+ - config HID_ASUS - tristate "Asus" - depends on USB_HID -@@ -741,6 +766,7 @@ config HID_MULTITOUCH - Say Y here if you have one of the following devices: - - 3M PCT touch screens - - ActionStar dual touch panels -+ - Touch Bars on x86 MacBook Pros - - Atmel panels - - Cando dual touch panels - - Chunghwa panels -diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile -index 24de45f3677d..1989288e0438 100644 ---- a/drivers/hid/Makefile -+++ b/drivers/hid/Makefile -@@ -29,6 +29,8 @@ obj-$(CONFIG_HID_ALPS) += hid-alps.o - obj-$(CONFIG_HID_ACRUX) += hid-axff.o - obj-$(CONFIG_HID_APPLE) += hid-apple.o - obj-$(CONFIG_HID_APPLEIR) += hid-appleir.o -+obj-$(CONFIG_HID_APPLETB_BL) += hid-appletb-bl.o -+obj-$(CONFIG_HID_APPLETB_KBD) += hid-appletb-kbd.o - obj-$(CONFIG_HID_CREATIVE_SB0540) += hid-creative-sb0540.o - obj-$(CONFIG_HID_ASUS) += hid-asus.o - obj-$(CONFIG_HID_AUREAL) += hid-aureal.o -diff --git a/drivers/hid/hid-appletb-bl.c b/drivers/hid/hid-appletb-bl.c -new file mode 100644 -index 000000000000..819157686e59 ---- /dev/null -+++ b/drivers/hid/hid-appletb-bl.c -@@ -0,0 +1,207 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* -+ * Apple Touch Bar Backlight Driver -+ * -+ * Copyright (c) 2017-2018 Ronald Tschalär -+ * Copyright (c) 2022-2023 Kerem Karabay -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+ -+#include "hid-ids.h" -+ -+#define APPLETB_BL_ON 1 -+#define APPLETB_BL_DIM 3 -+#define APPLETB_BL_OFF 4 -+ -+#define HID_UP_APPLEVENDOR_TB_BL 0xff120000 -+ -+#define HID_VD_APPLE_TB_BRIGHTNESS 0xff120001 -+#define HID_USAGE_AUX1 0xff120020 -+#define HID_USAGE_BRIGHTNESS 0xff120021 -+ -+static int appletb_bl_def_brightness = 2; -+module_param_named(brightness, appletb_bl_def_brightness, int, 0444); -+MODULE_PARM_DESC(brightness, "Default brightness:\n" -+ " 0 - Touchbar is off\n" -+ " 1 - Dim brightness\n" -+ " [2] - Full brightness"); -+ -+struct appletb_bl { -+ struct hid_field *aux1_field, *brightness_field; -+ struct backlight_device *bdev; -+ -+ bool full_on; -+}; -+ -+static const u8 appletb_bl_brightness_map[] = { -+ APPLETB_BL_OFF, -+ APPLETB_BL_DIM, -+ APPLETB_BL_ON, -+}; -+ -+static int appletb_bl_set_brightness(struct appletb_bl *bl, u8 brightness) -+{ -+ struct hid_report *report = bl->brightness_field->report; -+ struct hid_device *hdev = report->device; -+ int ret; -+ -+ ret = hid_set_field(bl->aux1_field, 0, 1); -+ if (ret) { -+ hid_err(hdev, "Failed to set auxiliary field (%pe)\n", ERR_PTR(ret)); -+ return ret; -+ } -+ -+ ret = hid_set_field(bl->brightness_field, 0, brightness); -+ if (ret) { -+ hid_err(hdev, "Failed to set brightness field (%pe)\n", ERR_PTR(ret)); -+ return ret; -+ } -+ -+ if (!bl->full_on) { -+ ret = hid_hw_power(hdev, PM_HINT_FULLON); -+ if (ret < 0) { -+ hid_err(hdev, "Device didn't power on (%pe)\n", ERR_PTR(ret)); -+ return ret; -+ } -+ -+ bl->full_on = true; -+ } -+ -+ hid_hw_request(hdev, report, HID_REQ_SET_REPORT); -+ -+ if (brightness == APPLETB_BL_OFF) { -+ hid_hw_power(hdev, PM_HINT_NORMAL); -+ bl->full_on = false; -+ } -+ -+ return 0; -+} -+ -+static int appletb_bl_update_status(struct backlight_device *bdev) -+{ -+ struct appletb_bl *bl = bl_get_data(bdev); -+ u8 brightness; -+ -+ if (backlight_is_blank(bdev)) -+ brightness = APPLETB_BL_OFF; -+ else -+ brightness = appletb_bl_brightness_map[backlight_get_brightness(bdev)]; -+ -+ return appletb_bl_set_brightness(bl, brightness); -+} -+ -+static const struct backlight_ops appletb_bl_backlight_ops = { -+ .options = BL_CORE_SUSPENDRESUME, -+ .update_status = appletb_bl_update_status, -+}; -+ -+static int appletb_bl_probe(struct hid_device *hdev, const struct hid_device_id *id) -+{ -+ struct hid_field *aux1_field, *brightness_field; -+ struct backlight_properties bl_props = { 0 }; -+ struct device *dev = &hdev->dev; -+ struct appletb_bl *bl; -+ int ret; -+ -+ ret = hid_parse(hdev); -+ if (ret) -+ return dev_err_probe(dev, ret, "HID parse failed\n"); -+ -+ aux1_field = hid_find_field(hdev, HID_FEATURE_REPORT, -+ HID_VD_APPLE_TB_BRIGHTNESS, HID_USAGE_AUX1); -+ -+ brightness_field = hid_find_field(hdev, HID_FEATURE_REPORT, -+ HID_VD_APPLE_TB_BRIGHTNESS, HID_USAGE_BRIGHTNESS); -+ -+ if (!aux1_field || !brightness_field) -+ return -ENODEV; -+ -+ if (aux1_field->report != brightness_field->report) -+ return dev_err_probe(dev, -ENODEV, "Encountered unexpected report structure\n"); -+ -+ bl = devm_kzalloc(dev, sizeof(*bl), GFP_KERNEL); -+ if (!bl) -+ return -ENOMEM; -+ -+ ret = hid_hw_start(hdev, HID_CONNECT_DRIVER); -+ if (ret) -+ return dev_err_probe(dev, ret, "HID hardware start failed\n"); -+ -+ ret = hid_hw_open(hdev); -+ if (ret) { -+ dev_err_probe(dev, ret, "HID hardware open failed\n"); -+ goto stop_hw; -+ } -+ -+ bl->aux1_field = aux1_field; -+ bl->brightness_field = brightness_field; -+ -+ if (appletb_bl_def_brightness == 0) -+ ret = appletb_bl_set_brightness(bl, APPLETB_BL_OFF); -+ else if (appletb_bl_def_brightness == 1) -+ ret = appletb_bl_set_brightness(bl, APPLETB_BL_DIM); -+ else -+ ret = appletb_bl_set_brightness(bl, APPLETB_BL_ON); -+ -+ if (ret) { -+ dev_err_probe(dev, ret, "Failed to set touch bar brightness to off\n"); -+ goto close_hw; -+ } -+ -+ bl_props.type = BACKLIGHT_RAW; -+ bl_props.max_brightness = ARRAY_SIZE(appletb_bl_brightness_map) - 1; -+ -+ bl->bdev = devm_backlight_device_register(dev, "appletb_backlight", dev, bl, -+ &appletb_bl_backlight_ops, &bl_props); -+ if (IS_ERR(bl->bdev)) { -+ ret = PTR_ERR(bl->bdev); -+ dev_err_probe(dev, ret, "Failed to register backlight device\n"); -+ goto close_hw; -+ } -+ -+ hid_set_drvdata(hdev, bl); -+ -+ return 0; -+ -+close_hw: -+ hid_hw_close(hdev); -+stop_hw: -+ hid_hw_stop(hdev); -+ -+ return ret; -+} -+ -+static void appletb_bl_remove(struct hid_device *hdev) -+{ -+ struct appletb_bl *bl = hid_get_drvdata(hdev); -+ -+ appletb_bl_set_brightness(bl, APPLETB_BL_OFF); -+ -+ hid_hw_close(hdev); -+ hid_hw_stop(hdev); -+} -+ -+static const struct hid_device_id appletb_bl_hid_ids[] = { -+ /* MacBook Pro's 2018, 2019, with T2 chip: iBridge DFR Brightness */ -+ { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_TOUCHBAR_BACKLIGHT) }, -+ { } -+}; -+MODULE_DEVICE_TABLE(hid, appletb_bl_hid_ids); -+ -+static struct hid_driver appletb_bl_hid_driver = { -+ .name = "hid-appletb-bl", -+ .id_table = appletb_bl_hid_ids, -+ .probe = appletb_bl_probe, -+ .remove = appletb_bl_remove, -+}; -+module_hid_driver(appletb_bl_hid_driver); -+ -+MODULE_AUTHOR("Ronald Tschalär"); -+MODULE_AUTHOR("Kerem Karabay "); -+MODULE_DESCRIPTION("MacBookPro Touch Bar Backlight Driver"); -+MODULE_LICENSE("GPL"); -diff --git a/drivers/hid/hid-appletb-kbd.c b/drivers/hid/hid-appletb-kbd.c -new file mode 100644 -index 000000000000..fa28a691da6a ---- /dev/null -+++ b/drivers/hid/hid-appletb-kbd.c -@@ -0,0 +1,506 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* -+ * Apple Touch Bar Keyboard Mode Driver -+ * -+ * Copyright (c) 2017-2018 Ronald Tschalär -+ * Copyright (c) 2022-2023 Kerem Karabay -+ * Copyright (c) 2024 Aditya Garg -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "hid-ids.h" -+ -+#define APPLETB_KBD_MODE_ESC 0 -+#define APPLETB_KBD_MODE_FN 1 -+#define APPLETB_KBD_MODE_SPCL 2 -+#define APPLETB_KBD_MODE_OFF 3 -+#define APPLETB_KBD_MODE_MAX APPLETB_KBD_MODE_OFF -+ -+#define APPLETB_DEVID_KEYBOARD 1 -+#define APPLETB_DEVID_TRACKPAD 2 -+ -+#define HID_USAGE_MODE 0x00ff0004 -+ -+static int appletb_tb_def_mode = APPLETB_KBD_MODE_SPCL; -+module_param_named(mode, appletb_tb_def_mode, int, 0444); -+MODULE_PARM_DESC(mode, "Default touchbar mode:\n" -+ " 0 - escape key only\n" -+ " 1 - function-keys\n" -+ " [2] - special keys"); -+ -+static bool appletb_tb_fn_toggle = true; -+module_param_named(fntoggle, appletb_tb_fn_toggle, bool, 0644); -+MODULE_PARM_DESC(fntoggle, "Switch between Fn and media controls on pressing Fn key"); -+ -+static bool appletb_tb_autodim = true; -+module_param_named(autodim, appletb_tb_autodim, bool, 0644); -+MODULE_PARM_DESC(autodim, "Automatically dim and turn off the Touch Bar after some time"); -+ -+static int appletb_tb_dim_timeout = 60; -+module_param_named(dim_timeout, appletb_tb_dim_timeout, int, 0644); -+MODULE_PARM_DESC(dim_timeout, "Dim timeout in sec"); -+ -+static int appletb_tb_idle_timeout = 15; -+module_param_named(idle_timeout, appletb_tb_idle_timeout, int, 0644); -+MODULE_PARM_DESC(idle_timeout, "Idle timeout in sec"); -+ -+struct appletb_kbd { -+ struct hid_field *mode_field; -+ struct input_handler inp_handler; -+ struct input_handle kbd_handle; -+ struct input_handle tpd_handle; -+ struct backlight_device *backlight_dev; -+ struct timer_list inactivity_timer; -+ bool has_dimmed; -+ bool has_turned_off; -+ u8 saved_mode; -+ u8 current_mode; -+}; -+ -+static const struct key_entry appletb_kbd_keymap[] = { -+ { KE_KEY, KEY_ESC, { KEY_ESC } }, -+ { KE_KEY, KEY_F1, { KEY_BRIGHTNESSDOWN } }, -+ { KE_KEY, KEY_F2, { KEY_BRIGHTNESSUP } }, -+ { KE_KEY, KEY_F3, { KEY_RESERVED } }, -+ { KE_KEY, KEY_F4, { KEY_RESERVED } }, -+ { KE_KEY, KEY_F5, { KEY_KBDILLUMDOWN } }, -+ { KE_KEY, KEY_F6, { KEY_KBDILLUMUP } }, -+ { KE_KEY, KEY_F7, { KEY_PREVIOUSSONG } }, -+ { KE_KEY, KEY_F8, { KEY_PLAYPAUSE } }, -+ { KE_KEY, KEY_F9, { KEY_NEXTSONG } }, -+ { KE_KEY, KEY_F10, { KEY_MUTE } }, -+ { KE_KEY, KEY_F11, { KEY_VOLUMEDOWN } }, -+ { KE_KEY, KEY_F12, { KEY_VOLUMEUP } }, -+ { KE_END, 0 } -+}; -+ -+static int appletb_kbd_set_mode(struct appletb_kbd *kbd, u8 mode) -+{ -+ struct hid_report *report = kbd->mode_field->report; -+ struct hid_device *hdev = report->device; -+ int ret; -+ -+ ret = hid_hw_power(hdev, PM_HINT_FULLON); -+ if (ret) { -+ hid_err(hdev, "Device didn't resume (%pe)\n", ERR_PTR(ret)); -+ return ret; -+ } -+ -+ ret = hid_set_field(kbd->mode_field, 0, mode); -+ if (ret) { -+ hid_err(hdev, "Failed to set mode field to %u (%pe)\n", mode, ERR_PTR(ret)); -+ goto power_normal; -+ } -+ -+ hid_hw_request(hdev, report, HID_REQ_SET_REPORT); -+ -+ kbd->current_mode = mode; -+ -+power_normal: -+ hid_hw_power(hdev, PM_HINT_NORMAL); -+ -+ return ret; -+} -+ -+static ssize_t mode_show(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ struct appletb_kbd *kbd = dev_get_drvdata(dev); -+ -+ return sysfs_emit(buf, "%d\n", kbd->current_mode); -+} -+ -+static ssize_t mode_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t size) -+{ -+ struct appletb_kbd *kbd = dev_get_drvdata(dev); -+ u8 mode; -+ int ret; -+ -+ ret = kstrtou8(buf, 0, &mode); -+ if (ret) -+ return ret; -+ -+ if (mode > APPLETB_KBD_MODE_MAX) -+ return -EINVAL; -+ -+ ret = appletb_kbd_set_mode(kbd, mode); -+ -+ return ret < 0 ? ret : size; -+} -+static DEVICE_ATTR_RW(mode); -+ -+struct attribute *appletb_kbd_attrs[] = { -+ &dev_attr_mode.attr, -+ NULL -+}; -+ATTRIBUTE_GROUPS(appletb_kbd); -+ -+static int appletb_tb_key_to_slot(unsigned int code) -+{ -+ switch (code) { -+ case KEY_ESC: -+ return 0; -+ case KEY_F1 ... KEY_F10: -+ return code - KEY_F1 + 1; -+ case KEY_F11 ... KEY_F12: -+ return code - KEY_F11 + 11; -+ -+ default: -+ return -EINVAL; -+ } -+} -+ -+static void appletb_inactivity_timer(struct timer_list *t) -+{ -+ struct appletb_kbd *kbd = from_timer(kbd, t, inactivity_timer); -+ -+ if (kbd->backlight_dev && appletb_tb_autodim) { -+ if (!kbd->has_dimmed) { -+ backlight_device_set_brightness(kbd->backlight_dev, 1); -+ kbd->has_dimmed = true; -+ mod_timer(&kbd->inactivity_timer, jiffies + msecs_to_jiffies(appletb_tb_idle_timeout * 1000)); -+ } else if (!kbd->has_turned_off) { -+ backlight_device_set_brightness(kbd->backlight_dev, 0); -+ kbd->has_turned_off = true; -+ } -+ } -+} -+ -+static void reset_inactivity_timer(struct appletb_kbd *kbd) -+{ -+ if (kbd->backlight_dev && appletb_tb_autodim) { -+ if (kbd->has_dimmed || kbd->has_turned_off) { -+ backlight_device_set_brightness(kbd->backlight_dev, 2); -+ kbd->has_dimmed = false; -+ kbd->has_turned_off = false; -+ } -+ mod_timer(&kbd->inactivity_timer, jiffies + msecs_to_jiffies(appletb_tb_dim_timeout * 1000)); -+ } -+} -+ -+static int appletb_kbd_hid_event(struct hid_device *hdev, struct hid_field *field, -+ struct hid_usage *usage, __s32 value) -+{ -+ struct appletb_kbd *kbd = hid_get_drvdata(hdev); -+ struct key_entry *translation; -+ struct input_dev *input; -+ int slot; -+ -+ if ((usage->hid & HID_USAGE_PAGE) != HID_UP_KEYBOARD || usage->type != EV_KEY) -+ return 0; -+ -+ input = field->hidinput->input; -+ -+ /* -+ * Skip non-touch-bar keys. -+ * -+ * Either the touch bar itself or usbhid generate a slew of key-down -+ * events for all the meta keys. None of which we're at all interested -+ * in. -+ */ -+ slot = appletb_tb_key_to_slot(usage->code); -+ if (slot < 0) -+ return 0; -+ -+ reset_inactivity_timer(kbd); -+ -+ translation = sparse_keymap_entry_from_scancode(input, usage->code); -+ -+ if (translation && kbd->current_mode == APPLETB_KBD_MODE_SPCL) { -+ input_event(input, usage->type, translation->keycode, value); -+ -+ return 1; -+ } -+ -+ return kbd->current_mode == APPLETB_KBD_MODE_OFF; -+} -+ -+static void appletb_kbd_inp_event(struct input_handle *handle, unsigned int type, -+ unsigned int code, int value) -+{ -+ struct appletb_kbd *kbd = handle->private; -+ -+ reset_inactivity_timer(kbd); -+ -+ if (type == EV_KEY && code == KEY_FN && appletb_tb_fn_toggle) { -+ if (value == 1) { -+ kbd->saved_mode = kbd->current_mode; -+ if (kbd->current_mode == APPLETB_KBD_MODE_SPCL) -+ appletb_kbd_set_mode(kbd, APPLETB_KBD_MODE_FN); -+ else if (kbd->current_mode == APPLETB_KBD_MODE_FN) -+ appletb_kbd_set_mode(kbd, APPLETB_KBD_MODE_SPCL); -+ } else if (value == 0) { -+ if (kbd->saved_mode != kbd->current_mode) -+ appletb_kbd_set_mode(kbd, kbd->saved_mode); -+ } -+ } -+} -+ -+static int appletb_kbd_inp_connect(struct input_handler *handler, -+ struct input_dev *dev, -+ const struct input_device_id *id) -+{ -+ struct appletb_kbd *kbd = handler->private; -+ struct input_handle *handle; -+ int rc; -+ -+ if (id->driver_info == APPLETB_DEVID_KEYBOARD) { -+ handle = &kbd->kbd_handle; -+ handle->name = "tbkbd"; -+ } else if (id->driver_info == APPLETB_DEVID_TRACKPAD) { -+ handle = &kbd->tpd_handle; -+ handle->name = "tbtpd"; -+ } else { -+ return -ENOENT; -+ } -+ -+ if (handle->dev) -+ return -EEXIST; -+ -+ handle->open = 0; -+ handle->dev = input_get_device(dev); -+ handle->handler = handler; -+ handle->private = kbd; -+ -+ rc = input_register_handle(handle); -+ if (rc) -+ goto err_free_dev; -+ -+ rc = input_open_device(handle); -+ if (rc) -+ goto err_unregister_handle; -+ -+ return 0; -+ -+ err_unregister_handle: -+ input_unregister_handle(handle); -+ err_free_dev: -+ input_put_device(handle->dev); -+ handle->dev = NULL; -+ return rc; -+} -+ -+static void appletb_kbd_inp_disconnect(struct input_handle *handle) -+{ -+ input_close_device(handle); -+ input_unregister_handle(handle); -+ -+ input_put_device(handle->dev); -+ handle->dev = NULL; -+} -+ -+static int appletb_kbd_input_configured(struct hid_device *hdev, struct hid_input *hidinput) -+{ -+ int idx; -+ struct input_dev *input = hidinput->input; -+ -+ /* -+ * Clear various input capabilities that are blindly set by the hid -+ * driver (usbkbd.c) -+ */ -+ memset(input->evbit, 0, sizeof(input->evbit)); -+ memset(input->keybit, 0, sizeof(input->keybit)); -+ memset(input->ledbit, 0, sizeof(input->ledbit)); -+ -+ __set_bit(EV_REP, input->evbit); -+ -+ sparse_keymap_setup(input, appletb_kbd_keymap, NULL); -+ -+ for (idx = 0; appletb_kbd_keymap[idx].type != KE_END; idx++) -+ input_set_capability(input, EV_KEY, appletb_kbd_keymap[idx].code); -+ -+ return 0; -+} -+ -+static const struct input_device_id appletb_kbd_input_devices[] = { -+ { -+ .flags = INPUT_DEVICE_ID_MATCH_BUS | -+ INPUT_DEVICE_ID_MATCH_VENDOR | -+ INPUT_DEVICE_ID_MATCH_KEYBIT, -+ .bustype = BUS_USB, -+ .vendor = USB_VENDOR_ID_APPLE, -+ .keybit = { [BIT_WORD(KEY_FN)] = BIT_MASK(KEY_FN) }, -+ .driver_info = APPLETB_DEVID_KEYBOARD, -+ }, -+ { -+ .flags = INPUT_DEVICE_ID_MATCH_BUS | -+ INPUT_DEVICE_ID_MATCH_VENDOR | -+ INPUT_DEVICE_ID_MATCH_KEYBIT, -+ .bustype = BUS_USB, -+ .vendor = USB_VENDOR_ID_APPLE, -+ .keybit = { [BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH) }, -+ .driver_info = APPLETB_DEVID_TRACKPAD, -+ }, -+ { } -+}; -+ -+static bool appletb_kbd_match_internal_device(struct input_handler *handler, -+ struct input_dev *inp_dev) -+{ -+ struct device *dev = &inp_dev->dev; -+ -+ /* in kernel: dev && !is_usb_device(dev) */ -+ while (dev && !(dev->type && dev->type->name && -+ !strcmp(dev->type->name, "usb_device"))) -+ dev = dev->parent; -+ -+ /* -+ * Apple labels all their internal keyboards and trackpads as such, -+ * instead of maintaining an ever expanding list of product-id's we -+ * just look at the device's product name. -+ */ -+ if (dev) -+ return !!strstr(to_usb_device(dev)->product, "Internal Keyboard"); -+ -+ return false; -+} -+ -+static int appletb_kbd_probe(struct hid_device *hdev, const struct hid_device_id *id) -+{ -+ struct appletb_kbd *kbd; -+ struct device *dev = &hdev->dev; -+ struct hid_field *mode_field; -+ int ret; -+ -+ ret = hid_parse(hdev); -+ if (ret) -+ return dev_err_probe(dev, ret, "HID parse failed\n"); -+ -+ mode_field = hid_find_field(hdev, HID_OUTPUT_REPORT, -+ HID_GD_KEYBOARD, HID_USAGE_MODE); -+ if (!mode_field) -+ return -ENODEV; -+ -+ kbd = devm_kzalloc(dev, sizeof(*kbd), GFP_KERNEL); -+ if (!kbd) -+ return -ENOMEM; -+ -+ kbd->mode_field = mode_field; -+ -+ ret = hid_hw_start(hdev, HID_CONNECT_HIDINPUT); -+ if (ret) -+ return dev_err_probe(dev, ret, "HID hw start failed\n"); -+ -+ ret = hid_hw_open(hdev); -+ if (ret) { -+ dev_err_probe(dev, ret, "HID hw open failed\n"); -+ goto stop_hw; -+ } -+ -+ kbd->backlight_dev = backlight_device_get_by_name("appletb_backlight"); -+ if (!kbd->backlight_dev) -+ dev_err_probe(dev, ret, "Failed to get backlight device\n"); -+ else { -+ backlight_device_set_brightness(kbd->backlight_dev, 2); -+ timer_setup(&kbd->inactivity_timer, appletb_inactivity_timer, 0); -+ mod_timer(&kbd->inactivity_timer, jiffies + msecs_to_jiffies(appletb_tb_dim_timeout * 1000)); -+ } -+ -+ kbd->inp_handler.event = appletb_kbd_inp_event; -+ kbd->inp_handler.connect = appletb_kbd_inp_connect; -+ kbd->inp_handler.disconnect = appletb_kbd_inp_disconnect; -+ kbd->inp_handler.name = "appletb"; -+ kbd->inp_handler.id_table = appletb_kbd_input_devices; -+ kbd->inp_handler.match = appletb_kbd_match_internal_device; -+ kbd->inp_handler.private = kbd; -+ -+ ret = input_register_handler(&kbd->inp_handler); -+ if (ret) { -+ dev_err_probe(dev, ret, "Unable to register keyboard handler\n"); -+ goto close_hw; -+ } -+ -+ ret = appletb_kbd_set_mode(kbd, appletb_tb_def_mode); -+ if (ret) { -+ dev_err_probe(dev, ret, "Failed to set touchbar mode\n"); -+ goto close_hw; -+ } -+ -+ hid_set_drvdata(hdev, kbd); -+ -+ return 0; -+ -+close_hw: -+ hid_hw_close(hdev); -+stop_hw: -+ hid_hw_stop(hdev); -+ return ret; -+} -+ -+static void appletb_kbd_remove(struct hid_device *hdev) -+{ -+ struct appletb_kbd *kbd = hid_get_drvdata(hdev); -+ -+ appletb_kbd_set_mode(kbd, APPLETB_KBD_MODE_OFF); -+ -+ input_unregister_handler(&kbd->inp_handler); -+ del_timer_sync(&kbd->inactivity_timer); -+ -+ hid_hw_close(hdev); -+ hid_hw_stop(hdev); -+} -+ -+#ifdef CONFIG_PM -+static int appletb_kbd_suspend(struct hid_device *hdev, pm_message_t msg) -+{ -+ struct appletb_kbd *kbd = hid_get_drvdata(hdev); -+ -+ kbd->saved_mode = kbd->current_mode; -+ appletb_kbd_set_mode(kbd, APPLETB_KBD_MODE_OFF); -+ -+ return 0; -+} -+ -+static int appletb_kbd_reset_resume(struct hid_device *hdev) -+{ -+ struct appletb_kbd *kbd = hid_get_drvdata(hdev); -+ -+ appletb_kbd_set_mode(kbd, kbd->saved_mode); -+ -+ return 0; -+} -+#endif -+ -+static const struct hid_device_id appletb_kbd_hid_ids[] = { -+ /* MacBook Pro's 2018, 2019, with T2 chip: iBridge Display */ -+ { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_TOUCHBAR_DISPLAY) }, -+ { } -+}; -+MODULE_DEVICE_TABLE(hid, appletb_kbd_hid_ids); -+ -+static struct hid_driver appletb_kbd_hid_driver = { -+ .name = "hid-appletb-kbd", -+ .id_table = appletb_kbd_hid_ids, -+ .probe = appletb_kbd_probe, -+ .remove = appletb_kbd_remove, -+ .event = appletb_kbd_hid_event, -+ .input_configured = appletb_kbd_input_configured, -+#ifdef CONFIG_PM -+ .suspend = appletb_kbd_suspend, -+ .reset_resume = appletb_kbd_reset_resume, -+#endif -+ .driver.dev_groups = appletb_kbd_groups, -+}; -+module_hid_driver(appletb_kbd_hid_driver); -+ -+/* The backlight driver should be loaded before the keyboard driver is initialised*/ -+MODULE_SOFTDEP("pre: hid_appletb_bl"); -+ -+MODULE_AUTHOR("Ronald Tschalär"); -+MODULE_AUTHOR("Kerem Karabay "); -+MODULE_DESCRIPTION("MacBookPro Touch Bar Keyboard Mode Driver"); -+MODULE_LICENSE("GPL"); -diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c -index 65023bfe30ed..42815300081a 100644 ---- a/drivers/hid/hid-multitouch.c -+++ b/drivers/hid/hid-multitouch.c -@@ -73,6 +73,7 @@ MODULE_LICENSE("GPL"); - #define MT_QUIRK_FORCE_MULTI_INPUT BIT(20) - #define MT_QUIRK_DISABLE_WAKEUP BIT(21) - #define MT_QUIRK_ORIENTATION_INVERT BIT(22) -+#define MT_QUIRK_TOUCH_IS_TIPSTATE BIT(23) - - #define MT_INPUTMODE_TOUCHSCREEN 0x02 - #define MT_INPUTMODE_TOUCHPAD 0x03 -@@ -153,6 +154,7 @@ struct mt_class { - __s32 sn_height; /* Signal/noise ratio for height events */ - __s32 sn_pressure; /* Signal/noise ratio for pressure events */ - __u8 maxcontacts; -+ bool is_direct; /* true for touchscreens */ - bool is_indirect; /* true for touchpads */ - bool export_all_inputs; /* do not ignore mouse, keyboards, etc... */ - }; -@@ -220,6 +222,7 @@ static void mt_post_parse(struct mt_device *td, struct mt_application *app); - #define MT_CLS_GOOGLE 0x0111 - #define MT_CLS_RAZER_BLADE_STEALTH 0x0112 - #define MT_CLS_SMART_TECH 0x0113 -+#define MT_CLS_APPLE_TOUCHBAR 0x0114 - #define MT_CLS_SIS 0x0457 - - #define MT_DEFAULT_MAXCONTACT 10 -@@ -405,6 +408,13 @@ static const struct mt_class mt_classes[] = { - MT_QUIRK_CONTACT_CNT_ACCURATE | - MT_QUIRK_SEPARATE_APP_REPORT, - }, -+ { .name = MT_CLS_APPLE_TOUCHBAR, -+ .quirks = MT_QUIRK_HOVERING | -+ MT_QUIRK_TOUCH_IS_TIPSTATE | -+ MT_QUIRK_SLOT_IS_CONTACTID_MINUS_ONE, -+ .is_direct = true, -+ .maxcontacts = 11, -+ }, - { .name = MT_CLS_SIS, - .quirks = MT_QUIRK_NOT_SEEN_MEANS_UP | - MT_QUIRK_ALWAYS_VALID | -@@ -503,9 +513,6 @@ static void mt_feature_mapping(struct hid_device *hdev, - if (!td->maxcontacts && - field->logical_maximum <= MT_MAX_MAXCONTACT) - td->maxcontacts = field->logical_maximum; -- if (td->mtclass.maxcontacts) -- /* check if the maxcontacts is given by the class */ -- td->maxcontacts = td->mtclass.maxcontacts; - - break; - case HID_DG_BUTTONTYPE: -@@ -579,13 +586,13 @@ static struct mt_application *mt_allocate_application(struct mt_device *td, - mt_application->application = application; - INIT_LIST_HEAD(&mt_application->mt_usages); - -- if (application == HID_DG_TOUCHSCREEN) -+ if (application == HID_DG_TOUCHSCREEN && !td->mtclass.is_indirect) - mt_application->mt_flags |= INPUT_MT_DIRECT; - - /* - * Model touchscreens providing buttons as touchpads. - */ -- if (application == HID_DG_TOUCHPAD) { -+ if (application == HID_DG_TOUCHPAD && !td->mtclass.is_direct) { - mt_application->mt_flags |= INPUT_MT_POINTER; - td->inputmode_value = MT_INPUTMODE_TOUCHPAD; - } -@@ -649,7 +656,9 @@ static struct mt_report_data *mt_allocate_report_data(struct mt_device *td, - - if (field->logical == HID_DG_FINGER || td->hdev->group != HID_GROUP_MULTITOUCH_WIN_8) { - for (n = 0; n < field->report_count; n++) { -- if (field->usage[n].hid == HID_DG_CONTACTID) { -+ unsigned int hid = field->usage[n].hid; -+ -+ if (hid == HID_DG_CONTACTID || hid == HID_DG_TRANSDUCER_INDEX) { - rdata->is_mt_collection = true; - break; - } -@@ -821,6 +830,15 @@ static int mt_touch_input_mapping(struct hid_device *hdev, struct hid_input *hi, - - MT_STORE_FIELD(confidence_state); - return 1; -+ case HID_DG_TOUCH: -+ /* -+ * Legacy devices use TIPSWITCH and not TOUCH. -+ * Let's just ignore this field unless the quirk is set. -+ */ -+ if (!(cls->quirks & MT_QUIRK_TOUCH_IS_TIPSTATE)) -+ return -1; -+ -+ fallthrough; - case HID_DG_TIPSWITCH: - if (field->application != HID_GD_SYSTEM_MULTIAXIS) - input_set_capability(hi->input, -@@ -828,6 +846,7 @@ static int mt_touch_input_mapping(struct hid_device *hdev, struct hid_input *hi, - MT_STORE_FIELD(tip_state); - return 1; - case HID_DG_CONTACTID: -+ case HID_DG_TRANSDUCER_INDEX: - MT_STORE_FIELD(contactid); - app->touches_by_report++; - return 1; -@@ -883,10 +902,6 @@ static int mt_touch_input_mapping(struct hid_device *hdev, struct hid_input *hi, - case HID_DG_CONTACTMAX: - /* contact max are global to the report */ - return -1; -- case HID_DG_TOUCH: -- /* Legacy devices use TIPSWITCH and not TOUCH. -- * Let's just ignore this field. */ -- return -1; - } - /* let hid-input decide for the others */ - return 0; -@@ -1314,6 +1329,10 @@ static int mt_touch_input_configured(struct hid_device *hdev, - struct input_dev *input = hi->input; - int ret; - -+ /* check if the maxcontacts is given by the class */ -+ if (cls->maxcontacts) -+ td->maxcontacts = cls->maxcontacts; -+ - if (!td->maxcontacts) - td->maxcontacts = MT_DEFAULT_MAXCONTACT; - -@@ -1321,6 +1340,9 @@ static int mt_touch_input_configured(struct hid_device *hdev, - if (td->serial_maybe) - mt_post_parse_default_settings(td, app); - -+ if (cls->is_direct) -+ app->mt_flags |= INPUT_MT_DIRECT; -+ - if (cls->is_indirect) - app->mt_flags |= INPUT_MT_POINTER; - -@@ -1769,6 +1791,15 @@ static int mt_probe(struct hid_device *hdev, const struct hid_device_id *id) - } - } - -+ ret = hid_parse(hdev); -+ if (ret != 0) -+ return ret; -+ -+ if (mtclass->name == MT_CLS_APPLE_TOUCHBAR && -+ !hid_find_field(hdev, HID_INPUT_REPORT, -+ HID_DG_TOUCHPAD, HID_DG_TRANSDUCER_INDEX)) -+ return -ENODEV; -+ - td = devm_kzalloc(&hdev->dev, sizeof(struct mt_device), GFP_KERNEL); - if (!td) { - dev_err(&hdev->dev, "cannot allocate multitouch data\n"); -@@ -1816,10 +1847,6 @@ static int mt_probe(struct hid_device *hdev, const struct hid_device_id *id) - - timer_setup(&td->release_timer, mt_expired_timeout, 0); - -- ret = hid_parse(hdev); -- if (ret != 0) -- return ret; -- - if (mtclass->quirks & MT_QUIRK_FIX_CONST_CONTACT_ID) - mt_fix_const_fields(hdev, HID_DG_CONTACTID); - -@@ -2301,6 +2328,11 @@ static const struct hid_device_id mt_devices[] = { - MT_USB_DEVICE(USB_VENDOR_ID_XIROKU, - USB_DEVICE_ID_XIROKU_CSR2) }, - -+ /* Apple Touch Bars */ -+ { .driver_data = MT_CLS_APPLE_TOUCHBAR, -+ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, -+ USB_DEVICE_ID_APPLE_TOUCHBAR_DISPLAY) }, -+ - /* Google MT devices */ - { .driver_data = MT_CLS_GOOGLE, - HID_DEVICE(HID_BUS_ANY, HID_GROUP_ANY, USB_VENDOR_ID_GOOGLE, -diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c -index e0bbf0c6345d..7c576d6540fe 100644 ---- a/drivers/hid/hid-quirks.c -+++ b/drivers/hid/hid-quirks.c -@@ -328,8 +328,6 @@ static const struct hid_device_id hid_have_special_driver[] = { - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER1_TP_ONLY) }, - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_2021) }, - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_FINGERPRINT_2021) }, -- { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_TOUCHBAR_BACKLIGHT) }, -- { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_TOUCHBAR_DISPLAY) }, - #endif - #if IS_ENABLED(CONFIG_HID_APPLEIR) - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_IRCONTROL) }, -@@ -338,6 +336,12 @@ static const struct hid_device_id hid_have_special_driver[] = { - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_IRCONTROL4) }, - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_IRCONTROL5) }, - #endif -+#if IS_ENABLED(CONFIG_HID_APPLETB_BL) -+ { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_TOUCHBAR_BACKLIGHT) }, -+#endif -+#if IS_ENABLED(CONFIG_HID_APPLETB_KBD) -+ { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_TOUCHBAR_DISPLAY) }, -+#endif - #if IS_ENABLED(CONFIG_HID_ASUS) - { HID_I2C_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_I2C_KEYBOARD) }, - { HID_I2C_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_I2C_TOUCHPAD) }, -diff --git a/drivers/hwmon/applesmc.c b/drivers/hwmon/applesmc.c -index fc6d6a9053ce..698f44794453 100644 ---- a/drivers/hwmon/applesmc.c -+++ b/drivers/hwmon/applesmc.c -@@ -6,6 +6,7 @@ - * - * Copyright (C) 2007 Nicolas Boichat - * Copyright (C) 2010 Henrik Rydberg -+ * Copyright (C) 2019 Paul Pawlowski - * - * Based on hdaps.c driver: - * Copyright (C) 2005 Robert Love -@@ -18,7 +19,7 @@ - #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - - #include --#include -+#include - #include - #include - #include -@@ -35,12 +36,24 @@ - #include - - /* data port used by Apple SMC */ --#define APPLESMC_DATA_PORT 0x300 -+#define APPLESMC_DATA_PORT 0 - /* command/status port used by Apple SMC */ --#define APPLESMC_CMD_PORT 0x304 -+#define APPLESMC_CMD_PORT 4 - - #define APPLESMC_NR_PORTS 32 /* 0x300-0x31f */ - -+#define APPLESMC_IOMEM_KEY_DATA 0 -+#define APPLESMC_IOMEM_KEY_STATUS 0x4005 -+#define APPLESMC_IOMEM_KEY_NAME 0x78 -+#define APPLESMC_IOMEM_KEY_DATA_LEN 0x7D -+#define APPLESMC_IOMEM_KEY_SMC_ID 0x7E -+#define APPLESMC_IOMEM_KEY_CMD 0x7F -+#define APPLESMC_IOMEM_MIN_SIZE 0x4006 -+ -+#define APPLESMC_IOMEM_KEY_TYPE_CODE 0 -+#define APPLESMC_IOMEM_KEY_TYPE_DATA_LEN 5 -+#define APPLESMC_IOMEM_KEY_TYPE_FLAGS 6 -+ - #define APPLESMC_MAX_DATA_LENGTH 32 - - /* Apple SMC status bits */ -@@ -74,6 +87,7 @@ - #define FAN_ID_FMT "F%dID" /* r-o char[16] */ - - #define TEMP_SENSOR_TYPE "sp78" -+#define FLOAT_TYPE "flt " - - /* List of keys used to read/write fan speeds */ - static const char *const fan_speed_fmt[] = { -@@ -83,6 +97,7 @@ static const char *const fan_speed_fmt[] = { - "F%dSf", /* safe speed - not all models */ - "F%dTg", /* target speed (manual: rw) */ - }; -+#define FAN_MANUAL_FMT "F%dMd" - - #define INIT_TIMEOUT_MSECS 5000 /* wait up to 5s for device init ... */ - #define INIT_WAIT_MSECS 50 /* ... in 50ms increments */ -@@ -119,7 +134,7 @@ struct applesmc_entry { - }; - - /* Register lookup and registers common to all SMCs */ --static struct applesmc_registers { -+struct applesmc_registers { - struct mutex mutex; /* register read/write mutex */ - unsigned int key_count; /* number of SMC registers */ - unsigned int fan_count; /* number of fans */ -@@ -133,26 +148,38 @@ static struct applesmc_registers { - bool init_complete; /* true when fully initialized */ - struct applesmc_entry *cache; /* cached key entries */ - const char **index; /* temperature key index */ --} smcreg = { -- .mutex = __MUTEX_INITIALIZER(smcreg.mutex), - }; - --static const int debug; --static struct platform_device *pdev; --static s16 rest_x; --static s16 rest_y; --static u8 backlight_state[2]; -+struct applesmc_device { -+ struct acpi_device *dev; -+ struct device *ldev; -+ struct applesmc_registers reg; - --static struct device *hwmon_dev; --static struct input_dev *applesmc_idev; -+ bool port_base_set, iomem_base_set; -+ u16 port_base; -+ u8 *__iomem iomem_base; -+ u32 iomem_base_addr, iomem_base_size; - --/* -- * Last index written to key_at_index sysfs file, and value to use for all other -- * key_at_index_* sysfs files. -- */ --static unsigned int key_at_index; -+ s16 rest_x; -+ s16 rest_y; -+ -+ u8 backlight_state[2]; -+ -+ struct device *hwmon_dev; -+ struct input_dev *idev; -+ -+ /* -+ * Last index written to key_at_index sysfs file, and value to use for all other -+ * key_at_index_* sysfs files. -+ */ -+ unsigned int key_at_index; - --static struct workqueue_struct *applesmc_led_wq; -+ struct workqueue_struct *backlight_wq; -+ struct work_struct backlight_work; -+ struct led_classdev backlight_dev; -+}; -+ -+static const int debug; - - /* - * Wait for specific status bits with a mask on the SMC. -@@ -162,7 +189,7 @@ static struct workqueue_struct *applesmc_led_wq; - * run out past 500ms. - */ - --static int wait_status(u8 val, u8 mask) -+static int port_wait_status(struct applesmc_device *smc, u8 val, u8 mask) - { - u8 status; - int us; -@@ -170,7 +197,7 @@ static int wait_status(u8 val, u8 mask) - - us = APPLESMC_MIN_WAIT; - for (i = 0; i < 24 ; i++) { -- status = inb(APPLESMC_CMD_PORT); -+ status = inb(smc->port_base + APPLESMC_CMD_PORT); - if ((status & mask) == val) - return 0; - usleep_range(us, us * 2); -@@ -180,13 +207,13 @@ static int wait_status(u8 val, u8 mask) - return -EIO; - } - --/* send_byte - Write to SMC data port. Callers must hold applesmc_lock. */ -+/* port_send_byte - Write to SMC data port. Callers must hold applesmc_lock. */ - --static int send_byte(u8 cmd, u16 port) -+static int port_send_byte(struct applesmc_device *smc, u8 cmd, u16 port) - { - int status; - -- status = wait_status(0, SMC_STATUS_IB_CLOSED); -+ status = port_wait_status(smc, 0, SMC_STATUS_IB_CLOSED); - if (status) - return status; - /* -@@ -195,24 +222,25 @@ static int send_byte(u8 cmd, u16 port) - * this extra read may not happen if status returns both - * simultaneously and this would appear to be required. - */ -- status = wait_status(SMC_STATUS_BUSY, SMC_STATUS_BUSY); -+ status = port_wait_status(smc, SMC_STATUS_BUSY, SMC_STATUS_BUSY); - if (status) - return status; - -- outb(cmd, port); -+ outb(cmd, smc->port_base + port); - return 0; - } - --/* send_command - Write a command to the SMC. Callers must hold applesmc_lock. */ -+/* port_send_command - Write a command to the SMC. Callers must hold applesmc_lock. */ - --static int send_command(u8 cmd) -+static int port_send_command(struct applesmc_device *smc, u8 cmd) - { - int ret; - -- ret = wait_status(0, SMC_STATUS_IB_CLOSED); -+ ret = port_wait_status(smc, 0, SMC_STATUS_IB_CLOSED); - if (ret) - return ret; -- outb(cmd, APPLESMC_CMD_PORT); -+ -+ outb(cmd, smc->port_base + APPLESMC_CMD_PORT); - return 0; - } - -@@ -222,110 +250,304 @@ static int send_command(u8 cmd) - * If busy is stuck high after the command then the SMC is jammed. - */ - --static int smc_sane(void) -+static int port_smc_sane(struct applesmc_device *smc) - { - int ret; - -- ret = wait_status(0, SMC_STATUS_BUSY); -+ ret = port_wait_status(smc, 0, SMC_STATUS_BUSY); - if (!ret) - return ret; -- ret = send_command(APPLESMC_READ_CMD); -+ ret = port_send_command(smc, APPLESMC_READ_CMD); - if (ret) - return ret; -- return wait_status(0, SMC_STATUS_BUSY); -+ return port_wait_status(smc, 0, SMC_STATUS_BUSY); - } - --static int send_argument(const char *key) -+static int port_send_argument(struct applesmc_device *smc, const char *key) - { - int i; - - for (i = 0; i < 4; i++) -- if (send_byte(key[i], APPLESMC_DATA_PORT)) -+ if (port_send_byte(smc, key[i], APPLESMC_DATA_PORT)) - return -EIO; - return 0; - } - --static int read_smc(u8 cmd, const char *key, u8 *buffer, u8 len) -+static int port_read_smc(struct applesmc_device *smc, u8 cmd, const char *key, -+ u8 *buffer, u8 len) - { - u8 status, data = 0; - int i; - int ret; - -- ret = smc_sane(); -+ ret = port_smc_sane(smc); - if (ret) - return ret; - -- if (send_command(cmd) || send_argument(key)) { -+ if (port_send_command(smc, cmd) || port_send_argument(smc, key)) { - pr_warn("%.4s: read arg fail\n", key); - return -EIO; - } - - /* This has no effect on newer (2012) SMCs */ -- if (send_byte(len, APPLESMC_DATA_PORT)) { -+ if (port_send_byte(smc, len, APPLESMC_DATA_PORT)) { - pr_warn("%.4s: read len fail\n", key); - return -EIO; - } - - for (i = 0; i < len; i++) { -- if (wait_status(SMC_STATUS_AWAITING_DATA | SMC_STATUS_BUSY, -+ if (port_wait_status(smc, -+ SMC_STATUS_AWAITING_DATA | SMC_STATUS_BUSY, - SMC_STATUS_AWAITING_DATA | SMC_STATUS_BUSY)) { - pr_warn("%.4s: read data[%d] fail\n", key, i); - return -EIO; - } -- buffer[i] = inb(APPLESMC_DATA_PORT); -+ buffer[i] = inb(smc->port_base + APPLESMC_DATA_PORT); - } - - /* Read the data port until bit0 is cleared */ - for (i = 0; i < 16; i++) { - udelay(APPLESMC_MIN_WAIT); -- status = inb(APPLESMC_CMD_PORT); -+ status = inb(smc->port_base + APPLESMC_CMD_PORT); - if (!(status & SMC_STATUS_AWAITING_DATA)) - break; -- data = inb(APPLESMC_DATA_PORT); -+ data = inb(smc->port_base + APPLESMC_DATA_PORT); - } - if (i) - pr_warn("flushed %d bytes, last value is: %d\n", i, data); - -- return wait_status(0, SMC_STATUS_BUSY); -+ return port_wait_status(smc, 0, SMC_STATUS_BUSY); - } - --static int write_smc(u8 cmd, const char *key, const u8 *buffer, u8 len) -+static int port_write_smc(struct applesmc_device *smc, u8 cmd, const char *key, -+ const u8 *buffer, u8 len) - { - int i; - int ret; - -- ret = smc_sane(); -+ ret = port_smc_sane(smc); - if (ret) - return ret; - -- if (send_command(cmd) || send_argument(key)) { -+ if (port_send_command(smc, cmd) || port_send_argument(smc, key)) { - pr_warn("%s: write arg fail\n", key); - return -EIO; - } - -- if (send_byte(len, APPLESMC_DATA_PORT)) { -+ if (port_send_byte(smc, len, APPLESMC_DATA_PORT)) { - pr_warn("%.4s: write len fail\n", key); - return -EIO; - } - - for (i = 0; i < len; i++) { -- if (send_byte(buffer[i], APPLESMC_DATA_PORT)) { -+ if (port_send_byte(smc, buffer[i], APPLESMC_DATA_PORT)) { - pr_warn("%s: write data fail\n", key); - return -EIO; - } - } - -- return wait_status(0, SMC_STATUS_BUSY); -+ return port_wait_status(smc, 0, SMC_STATUS_BUSY); - } - --static int read_register_count(unsigned int *count) -+static int port_get_smc_key_info(struct applesmc_device *smc, -+ const char *key, struct applesmc_entry *info) - { -- __be32 be; - int ret; -+ u8 raw[6]; - -- ret = read_smc(APPLESMC_READ_CMD, KEY_COUNT_KEY, (u8 *)&be, 4); -+ ret = port_read_smc(smc, APPLESMC_GET_KEY_TYPE_CMD, key, raw, 6); - if (ret) - return ret; -+ info->len = raw[0]; -+ memcpy(info->type, &raw[1], 4); -+ info->flags = raw[5]; -+ return 0; -+} -+ -+ -+/* -+ * MMIO based communication. -+ * TODO: Use updated mechanism for cmd timeout/retry -+ */ -+ -+static void iomem_clear_status(struct applesmc_device *smc) -+{ -+ if (ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_STATUS)) -+ iowrite8(0, smc->iomem_base + APPLESMC_IOMEM_KEY_STATUS); -+} -+ -+static int iomem_wait_read(struct applesmc_device *smc) -+{ -+ u8 status; -+ int us; -+ int i; -+ -+ us = APPLESMC_MIN_WAIT; -+ for (i = 0; i < 24 ; i++) { -+ status = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_STATUS); -+ if (status & 0x20) -+ return 0; -+ usleep_range(us, us * 2); -+ if (i > 9) -+ us <<= 1; -+ } -+ -+ dev_warn(smc->ldev, "%s... timeout\n", __func__); -+ return -EIO; -+} -+ -+static int iomem_read_smc(struct applesmc_device *smc, u8 cmd, const char *key, -+ u8 *buffer, u8 len) -+{ -+ u8 err, remote_len; -+ u32 key_int = *((u32 *) key); -+ -+ iomem_clear_status(smc); -+ iowrite32(key_int, smc->iomem_base + APPLESMC_IOMEM_KEY_NAME); -+ iowrite32(0, smc->iomem_base + APPLESMC_IOMEM_KEY_SMC_ID); -+ iowrite32(cmd, smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); -+ -+ if (iomem_wait_read(smc)) -+ return -EIO; -+ -+ err = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); -+ if (err != 0) { -+ dev_warn(smc->ldev, "read_smc_mmio(%x %8x/%.4s) failed: %u\n", -+ cmd, key_int, key, err); -+ return -EIO; -+ } -+ -+ if (cmd == APPLESMC_READ_CMD) { -+ remote_len = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_DATA_LEN); -+ if (remote_len != len) { -+ dev_warn(smc->ldev, -+ "read_smc_mmio(%x %8x/%.4s) failed: buffer length mismatch (remote = %u, requested = %u)\n", -+ cmd, key_int, key, remote_len, len); -+ return -EINVAL; -+ } -+ } else { -+ remote_len = len; -+ } -+ -+ memcpy_fromio(buffer, smc->iomem_base + APPLESMC_IOMEM_KEY_DATA, -+ remote_len); -+ -+ dev_dbg(smc->ldev, "read_smc_mmio(%x %8x/%.4s): buflen=%u reslen=%u\n", -+ cmd, key_int, key, len, remote_len); -+ print_hex_dump_bytes("read_smc_mmio(): ", DUMP_PREFIX_NONE, buffer, remote_len); -+ return 0; -+} -+ -+static int iomem_get_smc_key_type(struct applesmc_device *smc, const char *key, -+ struct applesmc_entry *e) -+{ -+ u8 err; -+ u8 cmd = APPLESMC_GET_KEY_TYPE_CMD; -+ u32 key_int = *((u32 *) key); -+ -+ iomem_clear_status(smc); -+ iowrite32(key_int, smc->iomem_base + APPLESMC_IOMEM_KEY_NAME); -+ iowrite32(0, smc->iomem_base + APPLESMC_IOMEM_KEY_SMC_ID); -+ iowrite32(cmd, smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); -+ -+ if (iomem_wait_read(smc)) -+ return -EIO; -+ -+ err = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); -+ if (err != 0) { -+ dev_warn(smc->ldev, "get_smc_key_type_mmio(%.4s) failed: %u\n", key, err); -+ return -EIO; -+ } -+ -+ e->len = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_TYPE_DATA_LEN); -+ *((uint32_t *) e->type) = ioread32( -+ smc->iomem_base + APPLESMC_IOMEM_KEY_TYPE_CODE); -+ e->flags = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_TYPE_FLAGS); -+ -+ dev_dbg(smc->ldev, "get_smc_key_type_mmio(%.4s): len=%u type=%.4s flags=%x\n", -+ key, e->len, e->type, e->flags); -+ return 0; -+} -+ -+static int iomem_write_smc(struct applesmc_device *smc, u8 cmd, const char *key, -+ const u8 *buffer, u8 len) -+{ -+ u8 err; -+ u32 key_int = *((u32 *) key); -+ -+ iomem_clear_status(smc); -+ iowrite32(key_int, smc->iomem_base + APPLESMC_IOMEM_KEY_NAME); -+ memcpy_toio(smc->iomem_base + APPLESMC_IOMEM_KEY_DATA, buffer, len); -+ iowrite32(len, smc->iomem_base + APPLESMC_IOMEM_KEY_DATA_LEN); -+ iowrite32(0, smc->iomem_base + APPLESMC_IOMEM_KEY_SMC_ID); -+ iowrite32(cmd, smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); -+ -+ if (iomem_wait_read(smc)) -+ return -EIO; -+ -+ err = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); -+ if (err != 0) { -+ dev_warn(smc->ldev, "write_smc_mmio(%x %.4s) failed: %u\n", cmd, key, err); -+ print_hex_dump_bytes("write_smc_mmio(): ", DUMP_PREFIX_NONE, buffer, len); -+ return -EIO; -+ } -+ -+ dev_dbg(smc->ldev, "write_smc_mmio(%x %.4s): buflen=%u\n", cmd, key, len); -+ print_hex_dump_bytes("write_smc_mmio(): ", DUMP_PREFIX_NONE, buffer, len); -+ return 0; -+} -+ -+ -+static int read_smc(struct applesmc_device *smc, const char *key, -+ u8 *buffer, u8 len) -+{ -+ if (smc->iomem_base_set) -+ return iomem_read_smc(smc, APPLESMC_READ_CMD, key, buffer, len); -+ else -+ return port_read_smc(smc, APPLESMC_READ_CMD, key, buffer, len); -+} -+ -+static int write_smc(struct applesmc_device *smc, const char *key, -+ const u8 *buffer, u8 len) -+{ -+ if (smc->iomem_base_set) -+ return iomem_write_smc(smc, APPLESMC_WRITE_CMD, key, buffer, len); -+ else -+ return port_write_smc(smc, APPLESMC_WRITE_CMD, key, buffer, len); -+} -+ -+static int get_smc_key_by_index(struct applesmc_device *smc, -+ unsigned int index, char *key) -+{ -+ __be32 be; -+ -+ be = cpu_to_be32(index); -+ if (smc->iomem_base_set) -+ return iomem_read_smc(smc, APPLESMC_GET_KEY_BY_INDEX_CMD, -+ (const char *) &be, (u8 *) key, 4); -+ else -+ return port_read_smc(smc, APPLESMC_GET_KEY_BY_INDEX_CMD, -+ (const char *) &be, (u8 *) key, 4); -+} -+ -+static int get_smc_key_info(struct applesmc_device *smc, const char *key, -+ struct applesmc_entry *info) -+{ -+ if (smc->iomem_base_set) -+ return iomem_get_smc_key_type(smc, key, info); -+ else -+ return port_get_smc_key_info(smc, key, info); -+} -+ -+static int read_register_count(struct applesmc_device *smc, -+ unsigned int *count) -+{ -+ __be32 be; -+ int ret; -+ -+ ret = read_smc(smc, KEY_COUNT_KEY, (u8 *)&be, 4); -+ if (ret < 0) -+ return ret; - - *count = be32_to_cpu(be); - return 0; -@@ -338,76 +560,73 @@ static int read_register_count(unsigned int *count) - * All functions below are concurrency safe - callers should NOT hold lock. - */ - --static int applesmc_read_entry(const struct applesmc_entry *entry, -- u8 *buf, u8 len) -+static int applesmc_read_entry(struct applesmc_device *smc, -+ const struct applesmc_entry *entry, u8 *buf, u8 len) - { - int ret; - - if (entry->len != len) - return -EINVAL; -- mutex_lock(&smcreg.mutex); -- ret = read_smc(APPLESMC_READ_CMD, entry->key, buf, len); -- mutex_unlock(&smcreg.mutex); -+ mutex_lock(&smc->reg.mutex); -+ ret = read_smc(smc, entry->key, buf, len); -+ mutex_unlock(&smc->reg.mutex); - - return ret; - } - --static int applesmc_write_entry(const struct applesmc_entry *entry, -- const u8 *buf, u8 len) -+static int applesmc_write_entry(struct applesmc_device *smc, -+ const struct applesmc_entry *entry, const u8 *buf, u8 len) - { - int ret; - - if (entry->len != len) - return -EINVAL; -- mutex_lock(&smcreg.mutex); -- ret = write_smc(APPLESMC_WRITE_CMD, entry->key, buf, len); -- mutex_unlock(&smcreg.mutex); -+ mutex_lock(&smc->reg.mutex); -+ ret = write_smc(smc, entry->key, buf, len); -+ mutex_unlock(&smc->reg.mutex); - return ret; - } - --static const struct applesmc_entry *applesmc_get_entry_by_index(int index) -+static const struct applesmc_entry *applesmc_get_entry_by_index( -+ struct applesmc_device *smc, int index) - { -- struct applesmc_entry *cache = &smcreg.cache[index]; -- u8 key[4], info[6]; -- __be32 be; -+ struct applesmc_entry *cache = &smc->reg.cache[index]; -+ char key[4]; - int ret = 0; - - if (cache->valid) - return cache; - -- mutex_lock(&smcreg.mutex); -+ mutex_lock(&smc->reg.mutex); - - if (cache->valid) - goto out; -- be = cpu_to_be32(index); -- ret = read_smc(APPLESMC_GET_KEY_BY_INDEX_CMD, (u8 *)&be, key, 4); -+ ret = get_smc_key_by_index(smc, index, key); - if (ret) - goto out; -- ret = read_smc(APPLESMC_GET_KEY_TYPE_CMD, key, info, 6); -+ memcpy(cache->key, key, 4); -+ -+ ret = get_smc_key_info(smc, key, cache); - if (ret) - goto out; -- -- memcpy(cache->key, key, 4); -- cache->len = info[0]; -- memcpy(cache->type, &info[1], 4); -- cache->flags = info[5]; - cache->valid = true; - - out: -- mutex_unlock(&smcreg.mutex); -+ mutex_unlock(&smc->reg.mutex); - if (ret) - return ERR_PTR(ret); - return cache; - } - --static int applesmc_get_lower_bound(unsigned int *lo, const char *key) -+static int applesmc_get_lower_bound(struct applesmc_device *smc, -+ unsigned int *lo, const char *key) - { -- int begin = 0, end = smcreg.key_count; -+ int begin = 0, end = smc->reg.key_count; - const struct applesmc_entry *entry; - - while (begin != end) { - int middle = begin + (end - begin) / 2; -- entry = applesmc_get_entry_by_index(middle); -+ entry = applesmc_get_entry_by_index(smc, middle); - if (IS_ERR(entry)) { - *lo = 0; - return PTR_ERR(entry); -@@ -422,16 +641,17 @@ static int applesmc_get_lower_bound(unsigned int *lo, const char *key) - return 0; - } - --static int applesmc_get_upper_bound(unsigned int *hi, const char *key) -+static int applesmc_get_upper_bound(struct applesmc_device *smc, -+ unsigned int *hi, const char *key) - { -- int begin = 0, end = smcreg.key_count; -+ int begin = 0, end = smc->reg.key_count; - const struct applesmc_entry *entry; - - while (begin != end) { - int middle = begin + (end - begin) / 2; -- entry = applesmc_get_entry_by_index(middle); -+ entry = applesmc_get_entry_by_index(smc, middle); - if (IS_ERR(entry)) { -- *hi = smcreg.key_count; -+ *hi = smc->reg.key_count; - return PTR_ERR(entry); - } - if (strcmp(key, entry->key) < 0) -@@ -444,50 +664,54 @@ static int applesmc_get_upper_bound(unsigned int *hi, const char *key) - return 0; - } - --static const struct applesmc_entry *applesmc_get_entry_by_key(const char *key) -+static const struct applesmc_entry *applesmc_get_entry_by_key( -+ struct applesmc_device *smc, const char *key) - { - int begin, end; - int ret; - -- ret = applesmc_get_lower_bound(&begin, key); -+ ret = applesmc_get_lower_bound(smc, &begin, key); - if (ret) - return ERR_PTR(ret); -- ret = applesmc_get_upper_bound(&end, key); -+ ret = applesmc_get_upper_bound(smc, &end, key); - if (ret) - return ERR_PTR(ret); - if (end - begin != 1) - return ERR_PTR(-EINVAL); - -- return applesmc_get_entry_by_index(begin); -+ return applesmc_get_entry_by_index(smc, begin); - } - --static int applesmc_read_key(const char *key, u8 *buffer, u8 len) -+static int applesmc_read_key(struct applesmc_device *smc, -+ const char *key, u8 *buffer, u8 len) - { - const struct applesmc_entry *entry; - -- entry = applesmc_get_entry_by_key(key); -+ entry = applesmc_get_entry_by_key(smc, key); - if (IS_ERR(entry)) - return PTR_ERR(entry); - -- return applesmc_read_entry(entry, buffer, len); -+ return applesmc_read_entry(smc, entry, buffer, len); - } - --static int applesmc_write_key(const char *key, const u8 *buffer, u8 len) -+static int applesmc_write_key(struct applesmc_device *smc, -+ const char *key, const u8 *buffer, u8 len) - { - const struct applesmc_entry *entry; - -- entry = applesmc_get_entry_by_key(key); -+ entry = applesmc_get_entry_by_key(smc, key); - if (IS_ERR(entry)) - return PTR_ERR(entry); - -- return applesmc_write_entry(entry, buffer, len); -+ return applesmc_write_entry(smc, entry, buffer, len); - } - --static int applesmc_has_key(const char *key, bool *value) -+static int applesmc_has_key(struct applesmc_device *smc, -+ const char *key, bool *value) - { - const struct applesmc_entry *entry; - -- entry = applesmc_get_entry_by_key(key); -+ entry = applesmc_get_entry_by_key(smc, key); - if (IS_ERR(entry) && PTR_ERR(entry) != -EINVAL) - return PTR_ERR(entry); - -@@ -498,12 +722,13 @@ static int applesmc_has_key(const char *key, bool *value) - /* - * applesmc_read_s16 - Read 16-bit signed big endian register - */ --static int applesmc_read_s16(const char *key, s16 *value) -+static int applesmc_read_s16(struct applesmc_device *smc, -+ const char *key, s16 *value) - { - u8 buffer[2]; - int ret; - -- ret = applesmc_read_key(key, buffer, 2); -+ ret = applesmc_read_key(smc, key, buffer, 2); - if (ret) - return ret; - -@@ -511,31 +736,68 @@ static int applesmc_read_s16(const char *key, s16 *value) - return 0; - } - -+/** -+ * applesmc_float_to_u32 - Retrieve the integral part of a float. -+ * This is needed because Apple made fans use float values in the T2. -+ * The fractional point is not significantly useful though, and the integral -+ * part can be easily extracted. -+ */ -+static inline u32 applesmc_float_to_u32(u32 d) -+{ -+ u8 sign = (u8) ((d >> 31) & 1); -+ s32 exp = (s32) ((d >> 23) & 0xff) - 0x7f; -+ u32 fr = d & ((1u << 23) - 1); -+ -+ if (sign || exp < 0) -+ return 0; -+ -+ return (u32) ((1u << exp) + (fr >> (23 - exp))); -+} -+ -+/** -+ * applesmc_u32_to_float - Convert an u32 into a float. -+ * See applesmc_float_to_u32 for a rationale. -+ */ -+static inline u32 applesmc_u32_to_float(u32 d) -+{ -+ u32 dc = d, bc = 0, exp; -+ -+ if (!d) -+ return 0; -+ -+ while (dc >>= 1) -+ ++bc; -+ exp = 0x7f + bc; -+ -+ return (u32) ((exp << 23) | -+ ((d << (23 - (exp - 0x7f))) & ((1u << 23) - 1))); -+} - /* - * applesmc_device_init - initialize the accelerometer. Can sleep. - */ --static void applesmc_device_init(void) -+static void applesmc_device_init(struct applesmc_device *smc) - { - int total; - u8 buffer[2]; - -- if (!smcreg.has_accelerometer) -+ if (!smc->reg.has_accelerometer) - return; - - for (total = INIT_TIMEOUT_MSECS; total > 0; total -= INIT_WAIT_MSECS) { -- if (!applesmc_read_key(MOTION_SENSOR_KEY, buffer, 2) && -+ if (!applesmc_read_key(smc, MOTION_SENSOR_KEY, buffer, 2) && - (buffer[0] != 0x00 || buffer[1] != 0x00)) - return; - buffer[0] = 0xe0; - buffer[1] = 0x00; -- applesmc_write_key(MOTION_SENSOR_KEY, buffer, 2); -+ applesmc_write_key(smc, MOTION_SENSOR_KEY, buffer, 2); - msleep(INIT_WAIT_MSECS); - } - - pr_warn("failed to init the device\n"); - } - --static int applesmc_init_index(struct applesmc_registers *s) -+static int applesmc_init_index(struct applesmc_device *smc, -+ struct applesmc_registers *s) - { - const struct applesmc_entry *entry; - unsigned int i; -@@ -548,7 +810,7 @@ static int applesmc_init_index(struct applesmc_registers *s) - return -ENOMEM; - - for (i = s->temp_begin; i < s->temp_end; i++) { -- entry = applesmc_get_entry_by_index(i); -+ entry = applesmc_get_entry_by_index(smc, i); - if (IS_ERR(entry)) - continue; - if (strcmp(entry->type, TEMP_SENSOR_TYPE)) -@@ -562,9 +824,9 @@ static int applesmc_init_index(struct applesmc_registers *s) - /* - * applesmc_init_smcreg_try - Try to initialize register cache. Idempotent. - */ --static int applesmc_init_smcreg_try(void) -+static int applesmc_init_smcreg_try(struct applesmc_device *smc) - { -- struct applesmc_registers *s = &smcreg; -+ struct applesmc_registers *s = &smc->reg; - bool left_light_sensor = false, right_light_sensor = false; - unsigned int count; - u8 tmp[1]; -@@ -573,7 +835,7 @@ static int applesmc_init_smcreg_try(void) - if (s->init_complete) - return 0; - -- ret = read_register_count(&count); -+ ret = read_register_count(smc, &count); - if (ret) - return ret; - -@@ -590,35 +852,35 @@ static int applesmc_init_smcreg_try(void) - if (!s->cache) - return -ENOMEM; - -- ret = applesmc_read_key(FANS_COUNT, tmp, 1); -+ ret = applesmc_read_key(smc, FANS_COUNT, tmp, 1); - if (ret) - return ret; - s->fan_count = tmp[0]; - if (s->fan_count > 10) - s->fan_count = 10; - -- ret = applesmc_get_lower_bound(&s->temp_begin, "T"); -+ ret = applesmc_get_lower_bound(smc, &s->temp_begin, "T"); - if (ret) - return ret; -- ret = applesmc_get_lower_bound(&s->temp_end, "U"); -+ ret = applesmc_get_lower_bound(smc, &s->temp_end, "U"); - if (ret) - return ret; - s->temp_count = s->temp_end - s->temp_begin; - -- ret = applesmc_init_index(s); -+ ret = applesmc_init_index(smc, s); - if (ret) - return ret; - -- ret = applesmc_has_key(LIGHT_SENSOR_LEFT_KEY, &left_light_sensor); -+ ret = applesmc_has_key(smc, LIGHT_SENSOR_LEFT_KEY, &left_light_sensor); - if (ret) - return ret; -- ret = applesmc_has_key(LIGHT_SENSOR_RIGHT_KEY, &right_light_sensor); -+ ret = applesmc_has_key(smc, LIGHT_SENSOR_RIGHT_KEY, &right_light_sensor); - if (ret) - return ret; -- ret = applesmc_has_key(MOTION_SENSOR_KEY, &s->has_accelerometer); -+ ret = applesmc_has_key(smc, MOTION_SENSOR_KEY, &s->has_accelerometer); - if (ret) - return ret; -- ret = applesmc_has_key(BACKLIGHT_KEY, &s->has_key_backlight); -+ ret = applesmc_has_key(smc, BACKLIGHT_KEY, &s->has_key_backlight); - if (ret) - return ret; - -@@ -634,13 +896,13 @@ static int applesmc_init_smcreg_try(void) - return 0; - } - --static void applesmc_destroy_smcreg(void) -+static void applesmc_destroy_smcreg(struct applesmc_device *smc) - { -- kfree(smcreg.index); -- smcreg.index = NULL; -- kfree(smcreg.cache); -- smcreg.cache = NULL; -- smcreg.init_complete = false; -+ kfree(smc->reg.index); -+ smc->reg.index = NULL; -+ kfree(smc->reg.cache); -+ smc->reg.cache = NULL; -+ smc->reg.init_complete = false; - } - - /* -@@ -649,12 +911,12 @@ static void applesmc_destroy_smcreg(void) - * Retries until initialization is successful, or the operation times out. - * - */ --static int applesmc_init_smcreg(void) -+static int applesmc_init_smcreg(struct applesmc_device *smc) - { - int ms, ret; - - for (ms = 0; ms < INIT_TIMEOUT_MSECS; ms += INIT_WAIT_MSECS) { -- ret = applesmc_init_smcreg_try(); -+ ret = applesmc_init_smcreg_try(smc); - if (!ret) { - if (ms) - pr_info("init_smcreg() took %d ms\n", ms); -@@ -663,50 +925,223 @@ static int applesmc_init_smcreg(void) - msleep(INIT_WAIT_MSECS); - } - -- applesmc_destroy_smcreg(); -+ applesmc_destroy_smcreg(smc); - - return ret; - } - - /* Device model stuff */ --static int applesmc_probe(struct platform_device *dev) -+ -+static int applesmc_init_resources(struct applesmc_device *smc); -+static void applesmc_free_resources(struct applesmc_device *smc); -+static int applesmc_create_modules(struct applesmc_device *smc); -+static void applesmc_destroy_modules(struct applesmc_device *smc); -+ -+static int applesmc_add(struct acpi_device *dev) - { -+ struct applesmc_device *smc; - int ret; - -- ret = applesmc_init_smcreg(); -+ smc = kzalloc(sizeof(struct applesmc_device), GFP_KERNEL); -+ if (!smc) -+ return -ENOMEM; -+ smc->dev = dev; -+ smc->ldev = &dev->dev; -+ mutex_init(&smc->reg.mutex); -+ -+ dev_set_drvdata(&dev->dev, smc); -+ -+ ret = applesmc_init_resources(smc); - if (ret) -- return ret; -+ goto out_mem; -+ -+ ret = applesmc_init_smcreg(smc); -+ if (ret) -+ goto out_res; -+ -+ applesmc_device_init(smc); -+ -+ ret = applesmc_create_modules(smc); -+ if (ret) -+ goto out_reg; -+ -+ return 0; -+ -+out_reg: -+ applesmc_destroy_smcreg(smc); -+out_res: -+ applesmc_free_resources(smc); -+out_mem: -+ dev_set_drvdata(&dev->dev, NULL); -+ mutex_destroy(&smc->reg.mutex); -+ kfree(smc); -+ -+ return ret; -+} -+ -+static void applesmc_remove(struct acpi_device *dev) -+{ -+ struct applesmc_device *smc = dev_get_drvdata(&dev->dev); -+ -+ applesmc_destroy_modules(smc); -+ applesmc_destroy_smcreg(smc); -+ applesmc_free_resources(smc); - -- applesmc_device_init(); -+ mutex_destroy(&smc->reg.mutex); -+ kfree(smc); -+ -+ return; -+} -+ -+static acpi_status applesmc_walk_resources(struct acpi_resource *res, -+ void *data) -+{ -+ struct applesmc_device *smc = data; -+ -+ switch (res->type) { -+ case ACPI_RESOURCE_TYPE_IO: -+ if (!smc->port_base_set) { -+ if (res->data.io.address_length < APPLESMC_NR_PORTS) -+ return AE_ERROR; -+ smc->port_base = res->data.io.minimum; -+ smc->port_base_set = true; -+ } -+ return AE_OK; -+ -+ case ACPI_RESOURCE_TYPE_FIXED_MEMORY32: -+ if (!smc->iomem_base_set) { -+ if (res->data.fixed_memory32.address_length < -+ APPLESMC_IOMEM_MIN_SIZE) { -+ dev_warn(smc->ldev, "found iomem but it's too small: %u\n", -+ res->data.fixed_memory32.address_length); -+ return AE_OK; -+ } -+ smc->iomem_base_addr = res->data.fixed_memory32.address; -+ smc->iomem_base_size = res->data.fixed_memory32.address_length; -+ smc->iomem_base_set = true; -+ } -+ return AE_OK; -+ -+ case ACPI_RESOURCE_TYPE_END_TAG: -+ if (smc->port_base_set) -+ return AE_OK; -+ else -+ return AE_NOT_FOUND; -+ -+ default: -+ return AE_OK; -+ } -+} -+ -+static int applesmc_try_enable_iomem(struct applesmc_device *smc); -+ -+static int applesmc_init_resources(struct applesmc_device *smc) -+{ -+ int ret; -+ -+ ret = acpi_walk_resources(smc->dev->handle, METHOD_NAME__CRS, -+ applesmc_walk_resources, smc); -+ if (ACPI_FAILURE(ret)) -+ return -ENXIO; -+ -+ if (!request_region(smc->port_base, APPLESMC_NR_PORTS, "applesmc")) -+ return -ENXIO; -+ -+ if (smc->iomem_base_set) { -+ if (applesmc_try_enable_iomem(smc)) -+ smc->iomem_base_set = false; -+ } -+ -+ return 0; -+} -+ -+static int applesmc_try_enable_iomem(struct applesmc_device *smc) -+{ -+ u8 test_val, ldkn_version; -+ -+ dev_dbg(smc->ldev, "Trying to enable iomem based communication\n"); -+ smc->iomem_base = ioremap(smc->iomem_base_addr, smc->iomem_base_size); -+ if (!smc->iomem_base) -+ goto out; -+ -+ /* Apple's driver does this check for some reason */ -+ test_val = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_STATUS); -+ if (test_val == 0xff) { -+ dev_warn(smc->ldev, -+ "iomem enable failed: initial status is 0xff (is %x)\n", -+ test_val); -+ goto out_iomem; -+ } -+ -+ if (read_smc(smc, "LDKN", &ldkn_version, 1)) { -+ dev_warn(smc->ldev, "iomem enable failed: ldkn read failed\n"); -+ goto out_iomem; -+ } -+ -+ if (ldkn_version < 2) { -+ dev_warn(smc->ldev, -+ "iomem enable failed: ldkn version %u is less than minimum (2)\n", -+ ldkn_version); -+ goto out_iomem; -+ } - - return 0; -+ -+out_iomem: -+ iounmap(smc->iomem_base); -+ -+out: -+ return -ENXIO; -+} -+ -+static void applesmc_free_resources(struct applesmc_device *smc) -+{ -+ if (smc->iomem_base_set) -+ iounmap(smc->iomem_base); -+ release_region(smc->port_base, APPLESMC_NR_PORTS); - } - - /* Synchronize device with memorized backlight state */ - static int applesmc_pm_resume(struct device *dev) - { -- if (smcreg.has_key_backlight) -- applesmc_write_key(BACKLIGHT_KEY, backlight_state, 2); -+ struct applesmc_device *smc = dev_get_drvdata(dev); -+ -+ if (smc->reg.has_key_backlight) -+ applesmc_write_key(smc, BACKLIGHT_KEY, smc->backlight_state, 2); -+ - return 0; - } - - /* Reinitialize device on resume from hibernation */ - static int applesmc_pm_restore(struct device *dev) - { -- applesmc_device_init(); -+ struct applesmc_device *smc = dev_get_drvdata(dev); -+ -+ applesmc_device_init(smc); -+ - return applesmc_pm_resume(dev); - } - -+static const struct acpi_device_id applesmc_ids[] = { -+ {"APP0001", 0}, -+ {"", 0}, -+}; -+ - static const struct dev_pm_ops applesmc_pm_ops = { - .resume = applesmc_pm_resume, - .restore = applesmc_pm_restore, - }; - --static struct platform_driver applesmc_driver = { -- .probe = applesmc_probe, -- .driver = { -- .name = "applesmc", -- .pm = &applesmc_pm_ops, -+static struct acpi_driver applesmc_driver = { -+ .name = "applesmc", -+ .class = "applesmc", -+ .ids = applesmc_ids, -+ .ops = { -+ .add = applesmc_add, -+ .remove = applesmc_remove -+ }, -+ .drv = { -+ .pm = &applesmc_pm_ops - }, - }; - -@@ -714,25 +1149,26 @@ static struct platform_driver applesmc_driver = { - * applesmc_calibrate - Set our "resting" values. Callers must - * hold applesmc_lock. - */ --static void applesmc_calibrate(void) -+static void applesmc_calibrate(struct applesmc_device *smc) - { -- applesmc_read_s16(MOTION_SENSOR_X_KEY, &rest_x); -- applesmc_read_s16(MOTION_SENSOR_Y_KEY, &rest_y); -- rest_x = -rest_x; -+ applesmc_read_s16(smc, MOTION_SENSOR_X_KEY, &smc->rest_x); -+ applesmc_read_s16(smc, MOTION_SENSOR_Y_KEY, &smc->rest_y); -+ smc->rest_x = -smc->rest_x; - } - - static void applesmc_idev_poll(struct input_dev *idev) - { -+ struct applesmc_device *smc = dev_get_drvdata(&idev->dev); - s16 x, y; - -- if (applesmc_read_s16(MOTION_SENSOR_X_KEY, &x)) -+ if (applesmc_read_s16(smc, MOTION_SENSOR_X_KEY, &x)) - return; -- if (applesmc_read_s16(MOTION_SENSOR_Y_KEY, &y)) -+ if (applesmc_read_s16(smc, MOTION_SENSOR_Y_KEY, &y)) - return; - - x = -x; -- input_report_abs(idev, ABS_X, x - rest_x); -- input_report_abs(idev, ABS_Y, y - rest_y); -+ input_report_abs(idev, ABS_X, x - smc->rest_x); -+ input_report_abs(idev, ABS_Y, y - smc->rest_y); - input_sync(idev); - } - -@@ -747,16 +1183,17 @@ static ssize_t applesmc_name_show(struct device *dev, - static ssize_t applesmc_position_show(struct device *dev, - struct device_attribute *attr, char *buf) - { -+ struct applesmc_device *smc = dev_get_drvdata(dev); - int ret; - s16 x, y, z; - -- ret = applesmc_read_s16(MOTION_SENSOR_X_KEY, &x); -+ ret = applesmc_read_s16(smc, MOTION_SENSOR_X_KEY, &x); - if (ret) - goto out; -- ret = applesmc_read_s16(MOTION_SENSOR_Y_KEY, &y); -+ ret = applesmc_read_s16(smc, MOTION_SENSOR_Y_KEY, &y); - if (ret) - goto out; -- ret = applesmc_read_s16(MOTION_SENSOR_Z_KEY, &z); -+ ret = applesmc_read_s16(smc, MOTION_SENSOR_Z_KEY, &z); - if (ret) - goto out; - -@@ -770,6 +1207,7 @@ static ssize_t applesmc_position_show(struct device *dev, - static ssize_t applesmc_light_show(struct device *dev, - struct device_attribute *attr, char *sysfsbuf) - { -+ struct applesmc_device *smc = dev_get_drvdata(dev); - const struct applesmc_entry *entry; - static int data_length; - int ret; -@@ -777,7 +1215,7 @@ static ssize_t applesmc_light_show(struct device *dev, - u8 buffer[10]; - - if (!data_length) { -- entry = applesmc_get_entry_by_key(LIGHT_SENSOR_LEFT_KEY); -+ entry = applesmc_get_entry_by_key(smc, LIGHT_SENSOR_LEFT_KEY); - if (IS_ERR(entry)) - return PTR_ERR(entry); - if (entry->len > 10) -@@ -786,7 +1224,7 @@ static ssize_t applesmc_light_show(struct device *dev, - pr_info("light sensor data length set to %d\n", data_length); - } - -- ret = applesmc_read_key(LIGHT_SENSOR_LEFT_KEY, buffer, data_length); -+ ret = applesmc_read_key(smc, LIGHT_SENSOR_LEFT_KEY, buffer, data_length); - if (ret) - goto out; - /* newer macbooks report a single 10-bit bigendian value */ -@@ -796,7 +1234,7 @@ static ssize_t applesmc_light_show(struct device *dev, - } - left = buffer[2]; - -- ret = applesmc_read_key(LIGHT_SENSOR_RIGHT_KEY, buffer, data_length); -+ ret = applesmc_read_key(smc, LIGHT_SENSOR_RIGHT_KEY, buffer, data_length); - if (ret) - goto out; - right = buffer[2]; -@@ -812,7 +1250,8 @@ static ssize_t applesmc_light_show(struct device *dev, - static ssize_t applesmc_show_sensor_label(struct device *dev, - struct device_attribute *devattr, char *sysfsbuf) - { -- const char *key = smcreg.index[to_index(devattr)]; -+ struct applesmc_device *smc = dev_get_drvdata(dev); -+ const char *key = smc->reg.index[to_index(devattr)]; - - return sysfs_emit(sysfsbuf, "%s\n", key); - } -@@ -821,12 +1260,13 @@ static ssize_t applesmc_show_sensor_label(struct device *dev, - static ssize_t applesmc_show_temperature(struct device *dev, - struct device_attribute *devattr, char *sysfsbuf) - { -- const char *key = smcreg.index[to_index(devattr)]; -+ struct applesmc_device *smc = dev_get_drvdata(dev); -+ const char *key = smc->reg.index[to_index(devattr)]; - int ret; - s16 value; - int temp; - -- ret = applesmc_read_s16(key, &value); -+ ret = applesmc_read_s16(smc, key, &value); - if (ret) - return ret; - -@@ -838,6 +1278,8 @@ static ssize_t applesmc_show_temperature(struct device *dev, - static ssize_t applesmc_show_fan_speed(struct device *dev, - struct device_attribute *attr, char *sysfsbuf) - { -+ struct applesmc_device *smc = dev_get_drvdata(dev); -+ const struct applesmc_entry *entry; - int ret; - unsigned int speed = 0; - char newkey[5]; -@@ -846,11 +1288,21 @@ static ssize_t applesmc_show_fan_speed(struct device *dev, - scnprintf(newkey, sizeof(newkey), fan_speed_fmt[to_option(attr)], - to_index(attr)); - -- ret = applesmc_read_key(newkey, buffer, 2); -+ entry = applesmc_get_entry_by_key(smc, newkey); -+ if (IS_ERR(entry)) -+ return PTR_ERR(entry); -+ -+ if (!strcmp(entry->type, FLOAT_TYPE)) { -+ ret = applesmc_read_entry(smc, entry, (u8 *) &speed, 4); -+ speed = applesmc_float_to_u32(speed); -+ } else { -+ ret = applesmc_read_entry(smc, entry, buffer, 2); -+ speed = ((buffer[0] << 8 | buffer[1]) >> 2); -+ } -+ - if (ret) - return ret; - -- speed = ((buffer[0] << 8 | buffer[1]) >> 2); - return sysfs_emit(sysfsbuf, "%u\n", speed); - } - -@@ -858,6 +1310,8 @@ static ssize_t applesmc_store_fan_speed(struct device *dev, - struct device_attribute *attr, - const char *sysfsbuf, size_t count) - { -+ struct applesmc_device *smc = dev_get_drvdata(dev); -+ const struct applesmc_entry *entry; - int ret; - unsigned long speed; - char newkey[5]; -@@ -869,9 +1323,18 @@ static ssize_t applesmc_store_fan_speed(struct device *dev, - scnprintf(newkey, sizeof(newkey), fan_speed_fmt[to_option(attr)], - to_index(attr)); - -- buffer[0] = (speed >> 6) & 0xff; -- buffer[1] = (speed << 2) & 0xff; -- ret = applesmc_write_key(newkey, buffer, 2); -+ entry = applesmc_get_entry_by_key(smc, newkey); -+ if (IS_ERR(entry)) -+ return PTR_ERR(entry); -+ -+ if (!strcmp(entry->type, FLOAT_TYPE)) { -+ speed = applesmc_u32_to_float(speed); -+ ret = applesmc_write_entry(smc, entry, (u8 *) &speed, 4); -+ } else { -+ buffer[0] = (speed >> 6) & 0xff; -+ buffer[1] = (speed << 2) & 0xff; -+ ret = applesmc_write_key(smc, newkey, buffer, 2); -+ } - - if (ret) - return ret; -@@ -882,15 +1345,30 @@ static ssize_t applesmc_store_fan_speed(struct device *dev, - static ssize_t applesmc_show_fan_manual(struct device *dev, - struct device_attribute *attr, char *sysfsbuf) - { -+ struct applesmc_device *smc = dev_get_drvdata(dev); - int ret; - u16 manual = 0; - u8 buffer[2]; -+ char newkey[5]; -+ bool has_newkey = false; -+ -+ scnprintf(newkey, sizeof(newkey), FAN_MANUAL_FMT, to_index(attr)); -+ -+ ret = applesmc_has_key(smc, newkey, &has_newkey); -+ if (ret) -+ return ret; -+ -+ if (has_newkey) { -+ ret = applesmc_read_key(smc, newkey, buffer, 1); -+ manual = buffer[0]; -+ } else { -+ ret = applesmc_read_key(smc, FANS_MANUAL, buffer, 2); -+ manual = ((buffer[0] << 8 | buffer[1]) >> to_index(attr)) & 0x01; -+ } - -- ret = applesmc_read_key(FANS_MANUAL, buffer, 2); - if (ret) - return ret; - -- manual = ((buffer[0] << 8 | buffer[1]) >> to_index(attr)) & 0x01; - return sysfs_emit(sysfsbuf, "%d\n", manual); - } - -@@ -898,29 +1376,42 @@ static ssize_t applesmc_store_fan_manual(struct device *dev, - struct device_attribute *attr, - const char *sysfsbuf, size_t count) - { -+ struct applesmc_device *smc = dev_get_drvdata(dev); - int ret; - u8 buffer[2]; -+ char newkey[5]; -+ bool has_newkey = false; - unsigned long input; - u16 val; - - if (kstrtoul(sysfsbuf, 10, &input) < 0) - return -EINVAL; - -- ret = applesmc_read_key(FANS_MANUAL, buffer, 2); -+ scnprintf(newkey, sizeof(newkey), FAN_MANUAL_FMT, to_index(attr)); -+ -+ ret = applesmc_has_key(smc, newkey, &has_newkey); - if (ret) -- goto out; -+ return ret; - -- val = (buffer[0] << 8 | buffer[1]); -+ if (has_newkey) { -+ buffer[0] = input & 1; -+ ret = applesmc_write_key(smc, newkey, buffer, 1); -+ } else { -+ ret = applesmc_read_key(smc, FANS_MANUAL, buffer, 2); -+ val = (buffer[0] << 8 | buffer[1]); -+ if (ret) -+ goto out; - -- if (input) -- val = val | (0x01 << to_index(attr)); -- else -- val = val & ~(0x01 << to_index(attr)); -+ if (input) -+ val = val | (0x01 << to_index(attr)); -+ else -+ val = val & ~(0x01 << to_index(attr)); - -- buffer[0] = (val >> 8) & 0xFF; -- buffer[1] = val & 0xFF; -+ buffer[0] = (val >> 8) & 0xFF; -+ buffer[1] = val & 0xFF; - -- ret = applesmc_write_key(FANS_MANUAL, buffer, 2); -+ ret = applesmc_write_key(smc, FANS_MANUAL, buffer, 2); -+ } - - out: - if (ret) -@@ -932,13 +1423,14 @@ static ssize_t applesmc_store_fan_manual(struct device *dev, - static ssize_t applesmc_show_fan_position(struct device *dev, - struct device_attribute *attr, char *sysfsbuf) - { -+ struct applesmc_device *smc = dev_get_drvdata(dev); - int ret; - char newkey[5]; - u8 buffer[17]; - - scnprintf(newkey, sizeof(newkey), FAN_ID_FMT, to_index(attr)); - -- ret = applesmc_read_key(newkey, buffer, 16); -+ ret = applesmc_read_key(smc, newkey, buffer, 16); - buffer[16] = 0; - - if (ret) -@@ -950,43 +1442,79 @@ static ssize_t applesmc_show_fan_position(struct device *dev, - static ssize_t applesmc_calibrate_show(struct device *dev, - struct device_attribute *attr, char *sysfsbuf) - { -- return sysfs_emit(sysfsbuf, "(%d,%d)\n", rest_x, rest_y); -+ struct applesmc_device *smc = dev_get_drvdata(dev); -+ -+ return sysfs_emit(sysfsbuf, "(%d,%d)\n", smc->rest_x, smc->rest_y); - } - - static ssize_t applesmc_calibrate_store(struct device *dev, - struct device_attribute *attr, const char *sysfsbuf, size_t count) - { -- applesmc_calibrate(); -+ struct applesmc_device *smc = dev_get_drvdata(dev); -+ -+ applesmc_calibrate(smc); - - return count; - } - - static void applesmc_backlight_set(struct work_struct *work) - { -- applesmc_write_key(BACKLIGHT_KEY, backlight_state, 2); -+ struct applesmc_device *smc = container_of(work, struct applesmc_device, backlight_work); -+ -+ applesmc_write_key(smc, BACKLIGHT_KEY, smc->backlight_state, 2); - } --static DECLARE_WORK(backlight_work, &applesmc_backlight_set); - - static void applesmc_brightness_set(struct led_classdev *led_cdev, - enum led_brightness value) - { -+ struct applesmc_device *smc = dev_get_drvdata(led_cdev->dev); - int ret; - -- backlight_state[0] = value; -- ret = queue_work(applesmc_led_wq, &backlight_work); -+ smc->backlight_state[0] = value; -+ ret = queue_work(smc->backlight_wq, &smc->backlight_work); - - if (debug && (!ret)) - dev_dbg(led_cdev->dev, "work was already on the queue.\n"); - } - -+static ssize_t applesmc_BCLM_store(struct device *dev, -+ struct device_attribute *attr, char *sysfsbuf, size_t count) -+{ -+ struct applesmc_device *smc = dev_get_drvdata(dev); -+ u8 val; -+ -+ if (kstrtou8(sysfsbuf, 10, &val) < 0) -+ return -EINVAL; -+ -+ if (val < 0 || val > 100) -+ return -EINVAL; -+ -+ if (applesmc_write_key(smc, "BCLM", &val, 1)) -+ return -ENODEV; -+ return count; -+} -+ -+static ssize_t applesmc_BCLM_show(struct device *dev, -+ struct device_attribute *attr, char *sysfsbuf) -+{ -+ struct applesmc_device *smc = dev_get_drvdata(dev); -+ u8 val; -+ -+ if (applesmc_read_key(smc, "BCLM", &val, 1)) -+ return -ENODEV; -+ -+ return sysfs_emit(sysfsbuf, "%d\n", val); -+} -+ - static ssize_t applesmc_key_count_show(struct device *dev, - struct device_attribute *attr, char *sysfsbuf) - { -+ struct applesmc_device *smc = dev_get_drvdata(dev); - int ret; - u8 buffer[4]; - u32 count; - -- ret = applesmc_read_key(KEY_COUNT_KEY, buffer, 4); -+ ret = applesmc_read_key(smc, KEY_COUNT_KEY, buffer, 4); - if (ret) - return ret; - -@@ -998,13 +1526,14 @@ static ssize_t applesmc_key_count_show(struct device *dev, - static ssize_t applesmc_key_at_index_read_show(struct device *dev, - struct device_attribute *attr, char *sysfsbuf) - { -+ struct applesmc_device *smc = dev_get_drvdata(dev); - const struct applesmc_entry *entry; - int ret; - -- entry = applesmc_get_entry_by_index(key_at_index); -+ entry = applesmc_get_entry_by_index(smc, smc->key_at_index); - if (IS_ERR(entry)) - return PTR_ERR(entry); -- ret = applesmc_read_entry(entry, sysfsbuf, entry->len); -+ ret = applesmc_read_entry(smc, entry, sysfsbuf, entry->len); - if (ret) - return ret; - -@@ -1014,9 +1543,10 @@ static ssize_t applesmc_key_at_index_read_show(struct device *dev, - static ssize_t applesmc_key_at_index_data_length_show(struct device *dev, - struct device_attribute *attr, char *sysfsbuf) - { -+ struct applesmc_device *smc = dev_get_drvdata(dev); - const struct applesmc_entry *entry; - -- entry = applesmc_get_entry_by_index(key_at_index); -+ entry = applesmc_get_entry_by_index(smc, smc->key_at_index); - if (IS_ERR(entry)) - return PTR_ERR(entry); - -@@ -1026,9 +1556,10 @@ static ssize_t applesmc_key_at_index_data_length_show(struct device *dev, - static ssize_t applesmc_key_at_index_type_show(struct device *dev, - struct device_attribute *attr, char *sysfsbuf) - { -+ struct applesmc_device *smc = dev_get_drvdata(dev); - const struct applesmc_entry *entry; - -- entry = applesmc_get_entry_by_index(key_at_index); -+ entry = applesmc_get_entry_by_index(smc, smc->key_at_index); - if (IS_ERR(entry)) - return PTR_ERR(entry); - -@@ -1038,9 +1569,10 @@ static ssize_t applesmc_key_at_index_type_show(struct device *dev, - static ssize_t applesmc_key_at_index_name_show(struct device *dev, - struct device_attribute *attr, char *sysfsbuf) - { -+ struct applesmc_device *smc = dev_get_drvdata(dev); - const struct applesmc_entry *entry; - -- entry = applesmc_get_entry_by_index(key_at_index); -+ entry = applesmc_get_entry_by_index(smc, smc->key_at_index); - if (IS_ERR(entry)) - return PTR_ERR(entry); - -@@ -1050,28 +1582,25 @@ static ssize_t applesmc_key_at_index_name_show(struct device *dev, - static ssize_t applesmc_key_at_index_show(struct device *dev, - struct device_attribute *attr, char *sysfsbuf) - { -- return sysfs_emit(sysfsbuf, "%d\n", key_at_index); -+ struct applesmc_device *smc = dev_get_drvdata(dev); -+ -+ return sysfs_emit(sysfsbuf, "%d\n", smc->key_at_index); - } - - static ssize_t applesmc_key_at_index_store(struct device *dev, - struct device_attribute *attr, const char *sysfsbuf, size_t count) - { -+ struct applesmc_device *smc = dev_get_drvdata(dev); - unsigned long newkey; - - if (kstrtoul(sysfsbuf, 10, &newkey) < 0 -- || newkey >= smcreg.key_count) -+ || newkey >= smc->reg.key_count) - return -EINVAL; - -- key_at_index = newkey; -+ smc->key_at_index = newkey; - return count; - } - --static struct led_classdev applesmc_backlight = { -- .name = "smc::kbd_backlight", -- .default_trigger = "nand-disk", -- .brightness_set = applesmc_brightness_set, --}; -- - static struct applesmc_node_group info_group[] = { - { "name", applesmc_name_show }, - { "key_count", applesmc_key_count_show }, -@@ -1111,19 +1640,25 @@ static struct applesmc_node_group temp_group[] = { - { } - }; - -+static struct applesmc_node_group BCLM_group[] = { -+ { "battery_charge_limit", applesmc_BCLM_show, applesmc_BCLM_store }, -+ { } -+}; -+ - /* Module stuff */ - - /* - * applesmc_destroy_nodes - remove files and free associated memory - */ --static void applesmc_destroy_nodes(struct applesmc_node_group *groups) -+static void applesmc_destroy_nodes(struct applesmc_device *smc, -+ struct applesmc_node_group *groups) - { - struct applesmc_node_group *grp; - struct applesmc_dev_attr *node; - - for (grp = groups; grp->nodes; grp++) { - for (node = grp->nodes; node->sda.dev_attr.attr.name; node++) -- sysfs_remove_file(&pdev->dev.kobj, -+ sysfs_remove_file(&smc->dev->dev.kobj, - &node->sda.dev_attr.attr); - kfree(grp->nodes); - grp->nodes = NULL; -@@ -1133,7 +1668,8 @@ static void applesmc_destroy_nodes(struct applesmc_node_group *groups) - /* - * applesmc_create_nodes - create a two-dimensional group of sysfs files - */ --static int applesmc_create_nodes(struct applesmc_node_group *groups, int num) -+static int applesmc_create_nodes(struct applesmc_device *smc, -+ struct applesmc_node_group *groups, int num) - { - struct applesmc_node_group *grp; - struct applesmc_dev_attr *node; -@@ -1157,7 +1693,7 @@ static int applesmc_create_nodes(struct applesmc_node_group *groups, int num) - sysfs_attr_init(attr); - attr->name = node->name; - attr->mode = 0444 | (grp->store ? 0200 : 0); -- ret = sysfs_create_file(&pdev->dev.kobj, attr); -+ ret = sysfs_create_file(&smc->dev->dev.kobj, attr); - if (ret) { - attr->name = NULL; - goto out; -@@ -1167,57 +1703,56 @@ static int applesmc_create_nodes(struct applesmc_node_group *groups, int num) - - return 0; - out: -- applesmc_destroy_nodes(groups); -+ applesmc_destroy_nodes(smc, groups); - return ret; - } - - /* Create accelerometer resources */ --static int applesmc_create_accelerometer(void) -+static int applesmc_create_accelerometer(struct applesmc_device *smc) - { - int ret; -- -- if (!smcreg.has_accelerometer) -+ if (!smc->reg.has_accelerometer) - return 0; - -- ret = applesmc_create_nodes(accelerometer_group, 1); -+ ret = applesmc_create_nodes(smc, accelerometer_group, 1); - if (ret) - goto out; - -- applesmc_idev = input_allocate_device(); -- if (!applesmc_idev) { -+ smc->idev = input_allocate_device(); -+ if (!smc->idev) { - ret = -ENOMEM; - goto out_sysfs; - } - - /* initial calibrate for the input device */ -- applesmc_calibrate(); -+ applesmc_calibrate(smc); - - /* initialize the input device */ -- applesmc_idev->name = "applesmc"; -- applesmc_idev->id.bustype = BUS_HOST; -- applesmc_idev->dev.parent = &pdev->dev; -- input_set_abs_params(applesmc_idev, ABS_X, -+ smc->idev->name = "applesmc"; -+ smc->idev->id.bustype = BUS_HOST; -+ smc->idev->dev.parent = &smc->dev->dev; -+ input_set_abs_params(smc->idev, ABS_X, - -256, 256, APPLESMC_INPUT_FUZZ, APPLESMC_INPUT_FLAT); -- input_set_abs_params(applesmc_idev, ABS_Y, -+ input_set_abs_params(smc->idev, ABS_Y, - -256, 256, APPLESMC_INPUT_FUZZ, APPLESMC_INPUT_FLAT); - -- ret = input_setup_polling(applesmc_idev, applesmc_idev_poll); -+ ret = input_setup_polling(smc->idev, applesmc_idev_poll); - if (ret) - goto out_idev; - -- input_set_poll_interval(applesmc_idev, APPLESMC_POLL_INTERVAL); -+ input_set_poll_interval(smc->idev, APPLESMC_POLL_INTERVAL); - -- ret = input_register_device(applesmc_idev); -+ ret = input_register_device(smc->idev); - if (ret) - goto out_idev; - - return 0; - - out_idev: -- input_free_device(applesmc_idev); -+ input_free_device(smc->idev); - - out_sysfs: -- applesmc_destroy_nodes(accelerometer_group); -+ applesmc_destroy_nodes(smc, accelerometer_group); - - out: - pr_warn("driver init failed (ret=%d)!\n", ret); -@@ -1225,44 +1760,55 @@ static int applesmc_create_accelerometer(void) - } - - /* Release all resources used by the accelerometer */ --static void applesmc_release_accelerometer(void) -+static void applesmc_release_accelerometer(struct applesmc_device *smc) - { -- if (!smcreg.has_accelerometer) -+ if (!smc->reg.has_accelerometer) - return; -- input_unregister_device(applesmc_idev); -- applesmc_destroy_nodes(accelerometer_group); -+ input_unregister_device(smc->idev); -+ applesmc_destroy_nodes(smc, accelerometer_group); - } - --static int applesmc_create_light_sensor(void) -+static int applesmc_create_light_sensor(struct applesmc_device *smc) - { -- if (!smcreg.num_light_sensors) -+ if (!smc->reg.num_light_sensors) - return 0; -- return applesmc_create_nodes(light_sensor_group, 1); -+ return applesmc_create_nodes(smc, light_sensor_group, 1); - } - --static void applesmc_release_light_sensor(void) -+static void applesmc_release_light_sensor(struct applesmc_device *smc) - { -- if (!smcreg.num_light_sensors) -+ if (!smc->reg.num_light_sensors) - return; -- applesmc_destroy_nodes(light_sensor_group); -+ applesmc_destroy_nodes(smc, light_sensor_group); - } - --static int applesmc_create_key_backlight(void) -+static int applesmc_create_key_backlight(struct applesmc_device *smc) - { -- if (!smcreg.has_key_backlight) -+ int ret; -+ -+ if (!smc->reg.has_key_backlight) - return 0; -- applesmc_led_wq = create_singlethread_workqueue("applesmc-led"); -- if (!applesmc_led_wq) -+ smc->backlight_wq = create_singlethread_workqueue("applesmc-led"); -+ if (!smc->backlight_wq) - return -ENOMEM; -- return led_classdev_register(&pdev->dev, &applesmc_backlight); -+ -+ INIT_WORK(&smc->backlight_work, applesmc_backlight_set); -+ smc->backlight_dev.name = "smc::kbd_backlight"; -+ smc->backlight_dev.default_trigger = "nand-disk"; -+ smc->backlight_dev.brightness_set = applesmc_brightness_set; -+ ret = led_classdev_register(&smc->dev->dev, &smc->backlight_dev); -+ if (ret) -+ destroy_workqueue(smc->backlight_wq); -+ -+ return ret; - } - --static void applesmc_release_key_backlight(void) -+static void applesmc_release_key_backlight(struct applesmc_device *smc) - { -- if (!smcreg.has_key_backlight) -+ if (!smc->reg.has_key_backlight) - return; -- led_classdev_unregister(&applesmc_backlight); -- destroy_workqueue(applesmc_led_wq); -+ led_classdev_unregister(&smc->backlight_dev); -+ destroy_workqueue(smc->backlight_wq); - } - - static int applesmc_dmi_match(const struct dmi_system_id *id) -@@ -1291,6 +1837,10 @@ static const struct dmi_system_id applesmc_whitelist[] __initconst = { - DMI_MATCH(DMI_BOARD_VENDOR, "Apple"), - DMI_MATCH(DMI_PRODUCT_NAME, "Macmini") }, - }, -+ { applesmc_dmi_match, "Apple iMacPro", { -+ DMI_MATCH(DMI_BOARD_VENDOR, "Apple"), -+ DMI_MATCH(DMI_PRODUCT_NAME, "iMacPro") }, -+ }, - { applesmc_dmi_match, "Apple MacPro", { - DMI_MATCH(DMI_BOARD_VENDOR, "Apple"), - DMI_MATCH(DMI_PRODUCT_NAME, "MacPro") }, -@@ -1306,90 +1856,91 @@ static const struct dmi_system_id applesmc_whitelist[] __initconst = { - { .ident = NULL } - }; - --static int __init applesmc_init(void) -+static int applesmc_create_modules(struct applesmc_device *smc) - { - int ret; - -- if (!dmi_check_system(applesmc_whitelist)) { -- pr_warn("supported laptop not found!\n"); -- ret = -ENODEV; -- goto out; -- } -- -- if (!request_region(APPLESMC_DATA_PORT, APPLESMC_NR_PORTS, -- "applesmc")) { -- ret = -ENXIO; -- goto out; -- } -- -- ret = platform_driver_register(&applesmc_driver); -- if (ret) -- goto out_region; -- -- pdev = platform_device_register_simple("applesmc", APPLESMC_DATA_PORT, -- NULL, 0); -- if (IS_ERR(pdev)) { -- ret = PTR_ERR(pdev); -- goto out_driver; -- } -- -- /* create register cache */ -- ret = applesmc_init_smcreg(); -+ ret = applesmc_create_nodes(smc, info_group, 1); - if (ret) -- goto out_device; -- -- ret = applesmc_create_nodes(info_group, 1); -+ goto out; -+ ret = applesmc_create_nodes(smc, BCLM_group, 1); - if (ret) -- goto out_smcreg; -+ goto out_info; - -- ret = applesmc_create_nodes(fan_group, smcreg.fan_count); -+ ret = applesmc_create_nodes(smc, fan_group, smc->reg.fan_count); - if (ret) -- goto out_info; -+ goto out_bclm; - -- ret = applesmc_create_nodes(temp_group, smcreg.index_count); -+ ret = applesmc_create_nodes(smc, temp_group, smc->reg.index_count); - if (ret) - goto out_fans; - -- ret = applesmc_create_accelerometer(); -+ ret = applesmc_create_accelerometer(smc); - if (ret) - goto out_temperature; - -- ret = applesmc_create_light_sensor(); -+ ret = applesmc_create_light_sensor(smc); - if (ret) - goto out_accelerometer; - -- ret = applesmc_create_key_backlight(); -+ ret = applesmc_create_key_backlight(smc); - if (ret) - goto out_light_sysfs; - -- hwmon_dev = hwmon_device_register(&pdev->dev); -- if (IS_ERR(hwmon_dev)) { -- ret = PTR_ERR(hwmon_dev); -+ smc->hwmon_dev = hwmon_device_register(&smc->dev->dev); -+ if (IS_ERR(smc->hwmon_dev)) { -+ ret = PTR_ERR(smc->hwmon_dev); - goto out_light_ledclass; - } - - return 0; - - out_light_ledclass: -- applesmc_release_key_backlight(); -+ applesmc_release_key_backlight(smc); - out_light_sysfs: -- applesmc_release_light_sensor(); -+ applesmc_release_light_sensor(smc); - out_accelerometer: -- applesmc_release_accelerometer(); -+ applesmc_release_accelerometer(smc); - out_temperature: -- applesmc_destroy_nodes(temp_group); -+ applesmc_destroy_nodes(smc, temp_group); - out_fans: -- applesmc_destroy_nodes(fan_group); -+ applesmc_destroy_nodes(smc, fan_group); -+out_bclm: -+ applesmc_destroy_nodes(smc, BCLM_group); - out_info: -- applesmc_destroy_nodes(info_group); --out_smcreg: -- applesmc_destroy_smcreg(); --out_device: -- platform_device_unregister(pdev); --out_driver: -- platform_driver_unregister(&applesmc_driver); --out_region: -- release_region(APPLESMC_DATA_PORT, APPLESMC_NR_PORTS); -+ applesmc_destroy_nodes(smc, info_group); -+out: -+ return ret; -+} -+ -+static void applesmc_destroy_modules(struct applesmc_device *smc) -+{ -+ hwmon_device_unregister(smc->hwmon_dev); -+ applesmc_release_key_backlight(smc); -+ applesmc_release_light_sensor(smc); -+ applesmc_release_accelerometer(smc); -+ applesmc_destroy_nodes(smc, temp_group); -+ applesmc_destroy_nodes(smc, fan_group); -+ applesmc_destroy_nodes(smc, BCLM_group); -+ applesmc_destroy_nodes(smc, info_group); -+} -+ -+static int __init applesmc_init(void) -+{ -+ int ret; -+ -+ if (!dmi_check_system(applesmc_whitelist)) { -+ pr_warn("supported laptop not found!\n"); -+ ret = -ENODEV; -+ goto out; -+ } -+ -+ ret = acpi_bus_register_driver(&applesmc_driver); -+ if (ret) -+ goto out; -+ -+ return 0; -+ - out: - pr_warn("driver init failed (ret=%d)!\n", ret); - return ret; -@@ -1397,23 +1948,14 @@ static int __init applesmc_init(void) - - static void __exit applesmc_exit(void) - { -- hwmon_device_unregister(hwmon_dev); -- applesmc_release_key_backlight(); -- applesmc_release_light_sensor(); -- applesmc_release_accelerometer(); -- applesmc_destroy_nodes(temp_group); -- applesmc_destroy_nodes(fan_group); -- applesmc_destroy_nodes(info_group); -- applesmc_destroy_smcreg(); -- platform_device_unregister(pdev); -- platform_driver_unregister(&applesmc_driver); -- release_region(APPLESMC_DATA_PORT, APPLESMC_NR_PORTS); -+ acpi_bus_unregister_driver(&applesmc_driver); - } - - module_init(applesmc_init); - module_exit(applesmc_exit); - - MODULE_AUTHOR("Nicolas Boichat"); -+MODULE_AUTHOR("Paul Pawlowski"); - MODULE_DESCRIPTION("Apple SMC"); - MODULE_LICENSE("GPL v2"); - MODULE_DEVICE_TABLE(dmi, applesmc_whitelist); -diff --git a/drivers/input/mouse/bcm5974.c b/drivers/input/mouse/bcm5974.c -index dfdfb59cc8b5..e0da70576167 100644 ---- a/drivers/input/mouse/bcm5974.c -+++ b/drivers/input/mouse/bcm5974.c -@@ -83,6 +83,24 @@ - #define USB_DEVICE_ID_APPLE_WELLSPRING9_ISO 0x0273 - #define USB_DEVICE_ID_APPLE_WELLSPRING9_JIS 0x0274 - -+/* T2-Attached Devices */ -+/* MacbookAir8,1 (2018) */ -+#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K 0x027a -+/* MacbookPro15,2 (2018) */ -+#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132 0x027b -+/* MacbookPro15,1 (2018) */ -+#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680 0x027c -+/* MacbookPro15,4 (2019) */ -+#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213 0x027d -+/* MacbookPro16,2 (2020) */ -+#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K 0x027e -+/* MacbookPro16,3 (2020) */ -+#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223 0x027f -+/* MacbookAir9,1 (2020) */ -+#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K 0x0280 -+/* MacbookPro16,1 (2019)*/ -+#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F 0x0340 -+ - #define BCM5974_DEVICE(prod) { \ - .match_flags = (USB_DEVICE_ID_MATCH_DEVICE | \ - USB_DEVICE_ID_MATCH_INT_CLASS | \ -@@ -147,6 +165,22 @@ static const struct usb_device_id bcm5974_table[] = { - BCM5974_DEVICE(USB_DEVICE_ID_APPLE_WELLSPRING9_ANSI), - BCM5974_DEVICE(USB_DEVICE_ID_APPLE_WELLSPRING9_ISO), - BCM5974_DEVICE(USB_DEVICE_ID_APPLE_WELLSPRING9_JIS), -+ /* MacbookAir8,1 */ -+ BCM5974_DEVICE(USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K), -+ /* MacbookPro15,2 */ -+ BCM5974_DEVICE(USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132), -+ /* MacbookPro15,1 */ -+ BCM5974_DEVICE(USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680), -+ /* MacbookPro15,4 */ -+ BCM5974_DEVICE(USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213), -+ /* MacbookPro16,2 */ -+ BCM5974_DEVICE(USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K), -+ /* MacbookPro16,3 */ -+ BCM5974_DEVICE(USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223), -+ /* MacbookAir9,1 */ -+ BCM5974_DEVICE(USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K), -+ /* MacbookPro16,1 */ -+ BCM5974_DEVICE(USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F), - /* Terminating entry */ - {} - }; -@@ -483,6 +517,110 @@ static const struct bcm5974_config bcm5974_config_table[] = { - { SN_COORD, -203, 6803 }, - { SN_ORIENT, -MAX_FINGER_ORIENTATION, MAX_FINGER_ORIENTATION } - }, -+ { -+ USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K, -+ 0, -+ 0, -+ HAS_INTEGRATED_BUTTON, -+ 0, sizeof(struct bt_data), -+ 0x83, DATAFORMAT(TYPE4), -+ { SN_PRESSURE, 0, 300 }, -+ { SN_WIDTH, 0, 2048 }, -+ { SN_COORD, -6243, 6749 }, -+ { SN_COORD, -170, 7685 }, -+ { SN_ORIENT, -MAX_FINGER_ORIENTATION, MAX_FINGER_ORIENTATION } -+ }, -+ { -+ USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132, -+ 0, -+ 0, -+ HAS_INTEGRATED_BUTTON, -+ 0, sizeof(struct bt_data), -+ 0x83, DATAFORMAT(TYPE4), -+ { SN_PRESSURE, 0, 300 }, -+ { SN_WIDTH, 0, 2048 }, -+ { SN_COORD, -6243, 6749 }, -+ { SN_COORD, -170, 7685 }, -+ { SN_ORIENT, -MAX_FINGER_ORIENTATION, MAX_FINGER_ORIENTATION } -+ }, -+ { -+ USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680, -+ 0, -+ 0, -+ HAS_INTEGRATED_BUTTON, -+ 0, sizeof(struct bt_data), -+ 0x83, DATAFORMAT(TYPE4), -+ { SN_PRESSURE, 0, 300 }, -+ { SN_WIDTH, 0, 2048 }, -+ { SN_COORD, -7456, 7976 }, -+ { SN_COORD, -1768, 7685 }, -+ { SN_ORIENT, -MAX_FINGER_ORIENTATION, MAX_FINGER_ORIENTATION } -+ }, -+ { -+ USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213, -+ 0, -+ 0, -+ HAS_INTEGRATED_BUTTON, -+ 0, sizeof(struct bt_data), -+ 0x83, DATAFORMAT(TYPE4), -+ { SN_PRESSURE, 0, 300 }, -+ { SN_WIDTH, 0, 2048 }, -+ { SN_COORD, -6243, 6749 }, -+ { SN_COORD, -170, 7685 }, -+ { SN_ORIENT, -MAX_FINGER_ORIENTATION, MAX_FINGER_ORIENTATION } -+ }, -+ { -+ USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K, -+ 0, -+ 0, -+ HAS_INTEGRATED_BUTTON, -+ 0, sizeof(struct bt_data), -+ 0x83, DATAFORMAT(TYPE4), -+ { SN_PRESSURE, 0, 300 }, -+ { SN_WIDTH, 0, 2048 }, -+ { SN_COORD, -7823, 8329 }, -+ { SN_COORD, -370, 7925 }, -+ { SN_ORIENT, -MAX_FINGER_ORIENTATION, MAX_FINGER_ORIENTATION } -+ }, -+ { -+ USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223, -+ 0, -+ 0, -+ HAS_INTEGRATED_BUTTON, -+ 0, sizeof(struct bt_data), -+ 0x83, DATAFORMAT(TYPE4), -+ { SN_PRESSURE, 0, 300 }, -+ { SN_WIDTH, 0, 2048 }, -+ { SN_COORD, -6243, 6749 }, -+ { SN_COORD, -170, 7685 }, -+ { SN_ORIENT, -MAX_FINGER_ORIENTATION, MAX_FINGER_ORIENTATION } -+ }, -+ { -+ USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K, -+ 0, -+ 0, -+ HAS_INTEGRATED_BUTTON, -+ 0, sizeof(struct bt_data), -+ 0x83, DATAFORMAT(TYPE4), -+ { SN_PRESSURE, 0, 300 }, -+ { SN_WIDTH, 0, 2048 }, -+ { SN_COORD, -6243, 6749 }, -+ { SN_COORD, -170, 7685 }, -+ { SN_ORIENT, -MAX_FINGER_ORIENTATION, MAX_FINGER_ORIENTATION } -+ }, -+ { -+ USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F, -+ 0, -+ 0, -+ HAS_INTEGRATED_BUTTON, -+ 0, sizeof(struct bt_data), -+ 0x83, DATAFORMAT(TYPE4), -+ { SN_PRESSURE, 0, 300 }, -+ { SN_WIDTH, 0, 2048 }, -+ { SN_COORD, -8916, 9918 }, -+ { SN_COORD, -1934, 9835 }, -+ { SN_ORIENT, -MAX_FINGER_ORIENTATION, MAX_FINGER_ORIENTATION } -+ }, - {} - }; - -diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c -index e4395b1f8c11..d2caa80e9412 100644 ---- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c -+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c -@@ -2712,7 +2712,7 @@ static const struct pci_device_id brcmf_pcie_devid_table[] = { - BRCMF_PCIE_DEVICE(BRCM_PCIE_4350_DEVICE_ID, WCC), - BRCMF_PCIE_DEVICE_SUB(0x4355, BRCM_PCIE_VENDOR_ID_BROADCOM, 0x4355, WCC), - BRCMF_PCIE_DEVICE(BRCM_PCIE_4354_RAW_DEVICE_ID, WCC), -- BRCMF_PCIE_DEVICE(BRCM_PCIE_4355_DEVICE_ID, WCC), -+ BRCMF_PCIE_DEVICE(BRCM_PCIE_4355_DEVICE_ID, WCC_SEED), - BRCMF_PCIE_DEVICE(BRCM_PCIE_4356_DEVICE_ID, WCC), - BRCMF_PCIE_DEVICE(BRCM_PCIE_43567_DEVICE_ID, WCC), - BRCMF_PCIE_DEVICE(BRCM_PCIE_43570_DEVICE_ID, WCC), -@@ -2723,7 +2723,7 @@ static const struct pci_device_id brcmf_pcie_devid_table[] = { - BRCMF_PCIE_DEVICE(BRCM_PCIE_43602_2G_DEVICE_ID, WCC), - BRCMF_PCIE_DEVICE(BRCM_PCIE_43602_5G_DEVICE_ID, WCC), - BRCMF_PCIE_DEVICE(BRCM_PCIE_43602_RAW_DEVICE_ID, WCC), -- BRCMF_PCIE_DEVICE(BRCM_PCIE_4364_DEVICE_ID, WCC), -+ BRCMF_PCIE_DEVICE(BRCM_PCIE_4364_DEVICE_ID, WCC_SEED), - BRCMF_PCIE_DEVICE(BRCM_PCIE_4365_DEVICE_ID, BCA), - BRCMF_PCIE_DEVICE(BRCM_PCIE_4365_2G_DEVICE_ID, BCA), - BRCMF_PCIE_DEVICE(BRCM_PCIE_4365_5G_DEVICE_ID, BCA), -diff --git a/drivers/pci/vgaarb.c b/drivers/pci/vgaarb.c -index 78748e8d2dba..2b2b558cebe6 100644 ---- a/drivers/pci/vgaarb.c -+++ b/drivers/pci/vgaarb.c -@@ -143,6 +143,7 @@ void vga_set_default_device(struct pci_dev *pdev) - pci_dev_put(vga_default); - vga_default = pci_dev_get(pdev); - } -+EXPORT_SYMBOL_GPL(vga_set_default_device); - - /** - * vga_remove_vgacon - deactivate VGA console -diff --git a/drivers/platform/x86/apple-gmux.c b/drivers/platform/x86/apple-gmux.c -index 1417e230edbd..e69785af8e1d 100644 ---- a/drivers/platform/x86/apple-gmux.c -+++ b/drivers/platform/x86/apple-gmux.c -@@ -21,6 +21,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -107,6 +108,10 @@ struct apple_gmux_config { - - # define MMIO_GMUX_MAX_BRIGHTNESS 0xffff - -+static bool force_igd; -+module_param(force_igd, bool, 0); -+MODULE_PARM_DESC(force_idg, "Switch gpu to igd on module load. Make sure that you have apple-set-os set up and the iGPU is in `lspci -s 00:02.0`. (default: false) (bool)"); -+ - static u8 gmux_pio_read8(struct apple_gmux_data *gmux_data, int port) - { - return inb(gmux_data->iostart + port); -@@ -945,6 +950,19 @@ static int gmux_probe(struct pnp_dev *pnp, const struct pnp_device_id *id) - gmux_enable_interrupts(gmux_data); - gmux_read_switch_state(gmux_data); - -+ if (force_igd) { -+ struct pci_dev *pdev; -+ -+ pdev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(2, 0)); -+ if (pdev) { -+ pr_info("Switching to IGD"); -+ gmux_switchto(VGA_SWITCHEROO_IGD); -+ vga_set_default_device(pdev); -+ } else { -+ pr_err("force_idg is true, but couldn't find iGPU at 00:02.0! Is apple-set-os working?"); -+ } -+ } -+ - /* - * Retina MacBook Pros cannot switch the panel's AUX separately - * and need eDP pre-calibration. They are distinguishable from -diff --git a/drivers/staging/Kconfig b/drivers/staging/Kconfig -index 075e775d3868..e1cc0d60eeb6 100644 ---- a/drivers/staging/Kconfig -+++ b/drivers/staging/Kconfig -@@ -50,4 +50,6 @@ source "drivers/staging/vme_user/Kconfig" - - source "drivers/staging/gpib/Kconfig" - -+source "drivers/staging/apple-bce/Kconfig" -+ - endif # STAGING -diff --git a/drivers/staging/Makefile b/drivers/staging/Makefile -index e681e403509c..4045c588b3b4 100644 ---- a/drivers/staging/Makefile -+++ b/drivers/staging/Makefile -@@ -14,3 +14,4 @@ obj-$(CONFIG_GREYBUS) += greybus/ - obj-$(CONFIG_BCM2835_VCHIQ) += vc04_services/ - obj-$(CONFIG_XIL_AXIS_FIFO) += axis-fifo/ - obj-$(CONFIG_GPIB) += gpib/ -+obj-$(CONFIG_APPLE_BCE) += apple-bce/ -diff --git a/drivers/staging/apple-bce/Kconfig b/drivers/staging/apple-bce/Kconfig -new file mode 100644 -index 000000000000..fe92bc441e89 ---- /dev/null -+++ b/drivers/staging/apple-bce/Kconfig -@@ -0,0 +1,18 @@ -+config APPLE_BCE -+ tristate "Apple BCE driver (VHCI and Audio support)" -+ default m -+ depends on X86 -+ select SOUND -+ select SND -+ select SND_PCM -+ select SND_JACK -+ help -+ VHCI and audio support on Apple MacBooks with the T2 Chip. -+ This driver is divided in three components: -+ - BCE (Buffer Copy Engine): which establishes a basic communication -+ channel with the T2 chip. This component is required by the other two: -+ - VHCI (Virtual Host Controller Interface): Access to keyboard, mouse -+ and other system devices depend on this virtual USB host controller -+ - Audio: a driver for the T2 audio interface. -+ -+ If "M" is selected, the module will be called apple-bce.' -diff --git a/drivers/staging/apple-bce/Makefile b/drivers/staging/apple-bce/Makefile -new file mode 100644 -index 000000000000..8cfbd3f64af6 ---- /dev/null -+++ b/drivers/staging/apple-bce/Makefile -@@ -0,0 +1,28 @@ -+modname := apple-bce -+obj-$(CONFIG_APPLE_BCE) += $(modname).o -+ -+apple-bce-objs := apple_bce.o mailbox.o queue.o queue_dma.o vhci/vhci.o vhci/queue.o vhci/transfer.o audio/audio.o audio/protocol.o audio/protocol_bce.o audio/pcm.o -+ -+MY_CFLAGS += -DWITHOUT_NVME_PATCH -+#MY_CFLAGS += -g -DDEBUG -+ccflags-y += ${MY_CFLAGS} -+CC += ${MY_CFLAGS} -+ -+KVERSION := $(KERNELRELEASE) -+ifeq ($(origin KERNELRELEASE), undefined) -+KVERSION := $(shell uname -r) -+endif -+ -+KDIR := /lib/modules/$(KVERSION)/build -+PWD := $(shell pwd) -+ -+.PHONY: all -+ -+all: -+ $(MAKE) -C $(KDIR) M=$(PWD) modules -+ -+clean: -+ $(MAKE) -C $(KDIR) M=$(PWD) clean -+ -+install: -+ $(MAKE) -C $(KDIR) M=$(PWD) modules_install -diff --git a/drivers/staging/apple-bce/apple_bce.c b/drivers/staging/apple-bce/apple_bce.c -new file mode 100644 -index 000000000000..4fd2415d7028 ---- /dev/null -+++ b/drivers/staging/apple-bce/apple_bce.c -@@ -0,0 +1,445 @@ -+#include "apple_bce.h" -+#include -+#include -+#include "audio/audio.h" -+#include -+ -+static dev_t bce_chrdev; -+static struct class *bce_class; -+ -+struct apple_bce_device *global_bce; -+ -+static int bce_create_command_queues(struct apple_bce_device *bce); -+static void bce_free_command_queues(struct apple_bce_device *bce); -+static irqreturn_t bce_handle_mb_irq(int irq, void *dev); -+static irqreturn_t bce_handle_dma_irq(int irq, void *dev); -+static int bce_fw_version_handshake(struct apple_bce_device *bce); -+static int bce_register_command_queue(struct apple_bce_device *bce, struct bce_queue_memcfg *cfg, int is_sq); -+ -+static int apple_bce_probe(struct pci_dev *dev, const struct pci_device_id *id) -+{ -+ struct apple_bce_device *bce = NULL; -+ int status = 0; -+ int nvec; -+ -+ pr_info("apple-bce: capturing our device\n"); -+ -+ if (pci_enable_device(dev)) -+ return -ENODEV; -+ if (pci_request_regions(dev, "apple-bce")) { -+ status = -ENODEV; -+ goto fail; -+ } -+ pci_set_master(dev); -+ nvec = pci_alloc_irq_vectors(dev, 1, 8, PCI_IRQ_MSI); -+ if (nvec < 5) { -+ status = -EINVAL; -+ goto fail; -+ } -+ -+ bce = kzalloc(sizeof(struct apple_bce_device), GFP_KERNEL); -+ if (!bce) { -+ status = -ENOMEM; -+ goto fail; -+ } -+ -+ bce->pci = dev; -+ pci_set_drvdata(dev, bce); -+ -+ bce->devt = bce_chrdev; -+ bce->dev = device_create(bce_class, &dev->dev, bce->devt, NULL, "apple-bce"); -+ if (IS_ERR_OR_NULL(bce->dev)) { -+ status = PTR_ERR(bce_class); -+ goto fail; -+ } -+ -+ bce->reg_mem_mb = pci_iomap(dev, 4, 0); -+ bce->reg_mem_dma = pci_iomap(dev, 2, 0); -+ -+ if (IS_ERR_OR_NULL(bce->reg_mem_mb) || IS_ERR_OR_NULL(bce->reg_mem_dma)) { -+ dev_warn(&dev->dev, "apple-bce: Failed to pci_iomap required regions\n"); -+ goto fail; -+ } -+ -+ bce_mailbox_init(&bce->mbox, bce->reg_mem_mb); -+ bce_timestamp_init(&bce->timestamp, bce->reg_mem_mb); -+ -+ spin_lock_init(&bce->queues_lock); -+ ida_init(&bce->queue_ida); -+ -+ if ((status = pci_request_irq(dev, 0, bce_handle_mb_irq, NULL, dev, "bce_mbox"))) -+ goto fail; -+ if ((status = pci_request_irq(dev, 4, NULL, bce_handle_dma_irq, dev, "bce_dma"))) -+ goto fail_interrupt_0; -+ -+ if ((status = dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(37)))) { -+ dev_warn(&dev->dev, "dma: Setting mask failed\n"); -+ goto fail_interrupt; -+ } -+ -+ /* Gets the function 0's interface. This is needed because Apple only accepts DMA on our function if function 0 -+ is a bus master, so we need to work around this. */ -+ bce->pci0 = pci_get_slot(dev->bus, PCI_DEVFN(PCI_SLOT(dev->devfn), 0)); -+#ifndef WITHOUT_NVME_PATCH -+ if ((status = pci_enable_device_mem(bce->pci0))) { -+ dev_warn(&dev->dev, "apple-bce: failed to enable function 0\n"); -+ goto fail_dev0; -+ } -+#endif -+ pci_set_master(bce->pci0); -+ -+ bce_timestamp_start(&bce->timestamp, true); -+ -+ if ((status = bce_fw_version_handshake(bce))) -+ goto fail_ts; -+ pr_info("apple-bce: handshake done\n"); -+ -+ if ((status = bce_create_command_queues(bce))) { -+ pr_info("apple-bce: Creating command queues failed\n"); -+ goto fail_ts; -+ } -+ -+ global_bce = bce; -+ -+ bce_vhci_create(bce, &bce->vhci); -+ -+ return 0; -+ -+fail_ts: -+ bce_timestamp_stop(&bce->timestamp); -+#ifndef WITHOUT_NVME_PATCH -+ pci_disable_device(bce->pci0); -+fail_dev0: -+#endif -+ pci_dev_put(bce->pci0); -+fail_interrupt: -+ pci_free_irq(dev, 4, dev); -+fail_interrupt_0: -+ pci_free_irq(dev, 0, dev); -+fail: -+ if (bce && bce->dev) { -+ device_destroy(bce_class, bce->devt); -+ -+ if (!IS_ERR_OR_NULL(bce->reg_mem_mb)) -+ pci_iounmap(dev, bce->reg_mem_mb); -+ if (!IS_ERR_OR_NULL(bce->reg_mem_dma)) -+ pci_iounmap(dev, bce->reg_mem_dma); -+ -+ kfree(bce); -+ } -+ -+ pci_free_irq_vectors(dev); -+ pci_release_regions(dev); -+ pci_disable_device(dev); -+ -+ if (!status) -+ status = -EINVAL; -+ return status; -+} -+ -+static int bce_create_command_queues(struct apple_bce_device *bce) -+{ -+ int status; -+ struct bce_queue_memcfg *cfg; -+ -+ bce->cmd_cq = bce_alloc_cq(bce, 0, 0x20); -+ bce->cmd_cmdq = bce_alloc_cmdq(bce, 1, 0x20); -+ if (bce->cmd_cq == NULL || bce->cmd_cmdq == NULL) { -+ status = -ENOMEM; -+ goto err; -+ } -+ bce->queues[0] = (struct bce_queue *) bce->cmd_cq; -+ bce->queues[1] = (struct bce_queue *) bce->cmd_cmdq->sq; -+ -+ cfg = kzalloc(sizeof(struct bce_queue_memcfg), GFP_KERNEL); -+ if (!cfg) { -+ status = -ENOMEM; -+ goto err; -+ } -+ bce_get_cq_memcfg(bce->cmd_cq, cfg); -+ if ((status = bce_register_command_queue(bce, cfg, false))) -+ goto err; -+ bce_get_sq_memcfg(bce->cmd_cmdq->sq, bce->cmd_cq, cfg); -+ if ((status = bce_register_command_queue(bce, cfg, true))) -+ goto err; -+ kfree(cfg); -+ -+ return 0; -+ -+err: -+ if (bce->cmd_cq) -+ bce_free_cq(bce, bce->cmd_cq); -+ if (bce->cmd_cmdq) -+ bce_free_cmdq(bce, bce->cmd_cmdq); -+ return status; -+} -+ -+static void bce_free_command_queues(struct apple_bce_device *bce) -+{ -+ bce_free_cq(bce, bce->cmd_cq); -+ bce_free_cmdq(bce, bce->cmd_cmdq); -+ bce->cmd_cq = NULL; -+ bce->queues[0] = NULL; -+} -+ -+static irqreturn_t bce_handle_mb_irq(int irq, void *dev) -+{ -+ struct apple_bce_device *bce = pci_get_drvdata(dev); -+ bce_mailbox_handle_interrupt(&bce->mbox); -+ return IRQ_HANDLED; -+} -+ -+static irqreturn_t bce_handle_dma_irq(int irq, void *dev) -+{ -+ int i; -+ struct apple_bce_device *bce = pci_get_drvdata(dev); -+ spin_lock(&bce->queues_lock); -+ for (i = 0; i < BCE_MAX_QUEUE_COUNT; i++) -+ if (bce->queues[i] && bce->queues[i]->type == BCE_QUEUE_CQ) -+ bce_handle_cq_completions(bce, (struct bce_queue_cq *) bce->queues[i]); -+ spin_unlock(&bce->queues_lock); -+ return IRQ_HANDLED; -+} -+ -+static int bce_fw_version_handshake(struct apple_bce_device *bce) -+{ -+ u64 result; -+ int status; -+ -+ if ((status = bce_mailbox_send(&bce->mbox, BCE_MB_MSG(BCE_MB_SET_FW_PROTOCOL_VERSION, BC_PROTOCOL_VERSION), -+ &result))) -+ return status; -+ if (BCE_MB_TYPE(result) != BCE_MB_SET_FW_PROTOCOL_VERSION || -+ BCE_MB_VALUE(result) != BC_PROTOCOL_VERSION) { -+ pr_err("apple-bce: FW version handshake failed %x:%llx\n", BCE_MB_TYPE(result), BCE_MB_VALUE(result)); -+ return -EINVAL; -+ } -+ return 0; -+} -+ -+static int bce_register_command_queue(struct apple_bce_device *bce, struct bce_queue_memcfg *cfg, int is_sq) -+{ -+ int status; -+ int cmd_type; -+ u64 result; -+ // OS X uses an bidirectional direction, but that's not really needed -+ dma_addr_t a = dma_map_single(&bce->pci->dev, cfg, sizeof(struct bce_queue_memcfg), DMA_TO_DEVICE); -+ if (dma_mapping_error(&bce->pci->dev, a)) -+ return -ENOMEM; -+ cmd_type = is_sq ? BCE_MB_REGISTER_COMMAND_SQ : BCE_MB_REGISTER_COMMAND_CQ; -+ status = bce_mailbox_send(&bce->mbox, BCE_MB_MSG(cmd_type, a), &result); -+ dma_unmap_single(&bce->pci->dev, a, sizeof(struct bce_queue_memcfg), DMA_TO_DEVICE); -+ if (status) -+ return status; -+ if (BCE_MB_TYPE(result) != BCE_MB_REGISTER_COMMAND_QUEUE_REPLY) -+ return -EINVAL; -+ return 0; -+} -+ -+static void apple_bce_remove(struct pci_dev *dev) -+{ -+ struct apple_bce_device *bce = pci_get_drvdata(dev); -+ bce->is_being_removed = true; -+ -+ bce_vhci_destroy(&bce->vhci); -+ -+ bce_timestamp_stop(&bce->timestamp); -+#ifndef WITHOUT_NVME_PATCH -+ pci_disable_device(bce->pci0); -+#endif -+ pci_dev_put(bce->pci0); -+ pci_free_irq(dev, 0, dev); -+ pci_free_irq(dev, 4, dev); -+ bce_free_command_queues(bce); -+ pci_iounmap(dev, bce->reg_mem_mb); -+ pci_iounmap(dev, bce->reg_mem_dma); -+ device_destroy(bce_class, bce->devt); -+ pci_free_irq_vectors(dev); -+ pci_release_regions(dev); -+ pci_disable_device(dev); -+ kfree(bce); -+} -+ -+static int bce_save_state_and_sleep(struct apple_bce_device *bce) -+{ -+ int attempt, status = 0; -+ u64 resp; -+ dma_addr_t dma_addr; -+ void *dma_ptr = NULL; -+ size_t size = max(PAGE_SIZE, 4096UL); -+ -+ for (attempt = 0; attempt < 5; ++attempt) { -+ pr_debug("apple-bce: suspend: attempt %i, buffer size %li\n", attempt, size); -+ dma_ptr = dma_alloc_coherent(&bce->pci->dev, size, &dma_addr, GFP_KERNEL); -+ if (!dma_ptr) { -+ pr_err("apple-bce: suspend failed (data alloc failed)\n"); -+ break; -+ } -+ BUG_ON((dma_addr % 4096) != 0); -+ status = bce_mailbox_send(&bce->mbox, -+ BCE_MB_MSG(BCE_MB_SAVE_STATE_AND_SLEEP, (dma_addr & ~(4096LLU - 1)) | (size / 4096)), &resp); -+ if (status) { -+ pr_err("apple-bce: suspend failed (mailbox send)\n"); -+ break; -+ } -+ if (BCE_MB_TYPE(resp) == BCE_MB_SAVE_RESTORE_STATE_COMPLETE) { -+ bce->saved_data_dma_addr = dma_addr; -+ bce->saved_data_dma_ptr = dma_ptr; -+ bce->saved_data_dma_size = size; -+ return 0; -+ } else if (BCE_MB_TYPE(resp) == BCE_MB_SAVE_STATE_AND_SLEEP_FAILURE) { -+ dma_free_coherent(&bce->pci->dev, size, dma_ptr, dma_addr); -+ /* The 0x10ff magic value was extracted from Apple's driver */ -+ size = (BCE_MB_VALUE(resp) + 0x10ff) & ~(4096LLU - 1); -+ pr_debug("apple-bce: suspend: device requested a larger buffer (%li)\n", size); -+ continue; -+ } else { -+ pr_err("apple-bce: suspend failed (invalid device response)\n"); -+ status = -EINVAL; -+ break; -+ } -+ } -+ if (dma_ptr) -+ dma_free_coherent(&bce->pci->dev, size, dma_ptr, dma_addr); -+ if (!status) -+ return bce_mailbox_send(&bce->mbox, BCE_MB_MSG(BCE_MB_SLEEP_NO_STATE, 0), &resp); -+ return status; -+} -+ -+static int bce_restore_state_and_wake(struct apple_bce_device *bce) -+{ -+ int status; -+ u64 resp; -+ if (!bce->saved_data_dma_ptr) { -+ if ((status = bce_mailbox_send(&bce->mbox, BCE_MB_MSG(BCE_MB_RESTORE_NO_STATE, 0), &resp))) { -+ pr_err("apple-bce: resume with no state failed (mailbox send)\n"); -+ return status; -+ } -+ if (BCE_MB_TYPE(resp) != BCE_MB_RESTORE_NO_STATE) { -+ pr_err("apple-bce: resume with no state failed (invalid device response)\n"); -+ return -EINVAL; -+ } -+ return 0; -+ } -+ -+ if ((status = bce_mailbox_send(&bce->mbox, BCE_MB_MSG(BCE_MB_RESTORE_STATE_AND_WAKE, -+ (bce->saved_data_dma_addr & ~(4096LLU - 1)) | (bce->saved_data_dma_size / 4096)), &resp))) { -+ pr_err("apple-bce: resume with state failed (mailbox send)\n"); -+ goto finish_with_state; -+ } -+ if (BCE_MB_TYPE(resp) != BCE_MB_SAVE_RESTORE_STATE_COMPLETE) { -+ pr_err("apple-bce: resume with state failed (invalid device response)\n"); -+ status = -EINVAL; -+ goto finish_with_state; -+ } -+ -+finish_with_state: -+ dma_free_coherent(&bce->pci->dev, bce->saved_data_dma_size, bce->saved_data_dma_ptr, bce->saved_data_dma_addr); -+ bce->saved_data_dma_ptr = NULL; -+ return status; -+} -+ -+static int apple_bce_suspend(struct device *dev) -+{ -+ struct apple_bce_device *bce = pci_get_drvdata(to_pci_dev(dev)); -+ int status; -+ -+ bce_timestamp_stop(&bce->timestamp); -+ -+ if ((status = bce_save_state_and_sleep(bce))) -+ return status; -+ -+ return 0; -+} -+ -+static int apple_bce_resume(struct device *dev) -+{ -+ struct apple_bce_device *bce = pci_get_drvdata(to_pci_dev(dev)); -+ int status; -+ -+ pci_set_master(bce->pci); -+ pci_set_master(bce->pci0); -+ -+ if ((status = bce_restore_state_and_wake(bce))) -+ return status; -+ -+ bce_timestamp_start(&bce->timestamp, false); -+ -+ return 0; -+} -+ -+static struct pci_device_id apple_bce_ids[ ] = { -+ { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1801) }, -+ { 0, }, -+}; -+ -+MODULE_DEVICE_TABLE(pci, apple_bce_ids); -+ -+struct dev_pm_ops apple_bce_pci_driver_pm = { -+ .suspend = apple_bce_suspend, -+ .resume = apple_bce_resume -+}; -+struct pci_driver apple_bce_pci_driver = { -+ .name = "apple-bce", -+ .id_table = apple_bce_ids, -+ .probe = apple_bce_probe, -+ .remove = apple_bce_remove, -+ .driver = { -+ .pm = &apple_bce_pci_driver_pm -+ } -+}; -+ -+ -+static int __init apple_bce_module_init(void) -+{ -+ int result; -+ if ((result = alloc_chrdev_region(&bce_chrdev, 0, 1, "apple-bce"))) -+ goto fail_chrdev; -+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,4,0) -+ bce_class = class_create(THIS_MODULE, "apple-bce"); -+#else -+ bce_class = class_create("apple-bce"); -+#endif -+ if (IS_ERR(bce_class)) { -+ result = PTR_ERR(bce_class); -+ goto fail_class; -+ } -+ if ((result = bce_vhci_module_init())) { -+ pr_err("apple-bce: bce-vhci init failed"); -+ goto fail_class; -+ } -+ -+ result = pci_register_driver(&apple_bce_pci_driver); -+ if (result) -+ goto fail_drv; -+ -+ aaudio_module_init(); -+ -+ return 0; -+ -+fail_drv: -+ pci_unregister_driver(&apple_bce_pci_driver); -+fail_class: -+ class_destroy(bce_class); -+fail_chrdev: -+ unregister_chrdev_region(bce_chrdev, 1); -+ if (!result) -+ result = -EINVAL; -+ return result; -+} -+static void __exit apple_bce_module_exit(void) -+{ -+ pci_unregister_driver(&apple_bce_pci_driver); -+ -+ aaudio_module_exit(); -+ bce_vhci_module_exit(); -+ class_destroy(bce_class); -+ unregister_chrdev_region(bce_chrdev, 1); -+} -+ -+MODULE_LICENSE("GPL"); -+MODULE_AUTHOR("MrARM"); -+MODULE_DESCRIPTION("Apple BCE Driver"); -+MODULE_VERSION("0.01"); -+module_init(apple_bce_module_init); -+module_exit(apple_bce_module_exit); -diff --git a/drivers/staging/apple-bce/apple_bce.h b/drivers/staging/apple-bce/apple_bce.h -new file mode 100644 -index 000000000000..f13ab8d5742e ---- /dev/null -+++ b/drivers/staging/apple-bce/apple_bce.h -@@ -0,0 +1,38 @@ -+#pragma once -+ -+#include -+#include -+#include "mailbox.h" -+#include "queue.h" -+#include "vhci/vhci.h" -+ -+#define BC_PROTOCOL_VERSION 0x20001 -+#define BCE_MAX_QUEUE_COUNT 0x100 -+ -+#define BCE_QUEUE_USER_MIN 2 -+#define BCE_QUEUE_USER_MAX (BCE_MAX_QUEUE_COUNT - 1) -+ -+struct apple_bce_device { -+ struct pci_dev *pci, *pci0; -+ dev_t devt; -+ struct device *dev; -+ void __iomem *reg_mem_mb; -+ void __iomem *reg_mem_dma; -+ struct bce_mailbox mbox; -+ struct bce_timestamp timestamp; -+ struct bce_queue *queues[BCE_MAX_QUEUE_COUNT]; -+ struct spinlock queues_lock; -+ struct ida queue_ida; -+ struct bce_queue_cq *cmd_cq; -+ struct bce_queue_cmdq *cmd_cmdq; -+ struct bce_queue_sq *int_sq_list[BCE_MAX_QUEUE_COUNT]; -+ bool is_being_removed; -+ -+ dma_addr_t saved_data_dma_addr; -+ void *saved_data_dma_ptr; -+ size_t saved_data_dma_size; -+ -+ struct bce_vhci vhci; -+}; -+ -+extern struct apple_bce_device *global_bce; -\ No newline at end of file -diff --git a/drivers/staging/apple-bce/audio/audio.c b/drivers/staging/apple-bce/audio/audio.c -new file mode 100644 -index 000000000000..bd16ddd16c1d ---- /dev/null -+++ b/drivers/staging/apple-bce/audio/audio.c -@@ -0,0 +1,711 @@ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include "audio.h" -+#include "pcm.h" -+#include -+ -+static int aaudio_alsa_index = SNDRV_DEFAULT_IDX1; -+static char *aaudio_alsa_id = SNDRV_DEFAULT_STR1; -+ -+static dev_t aaudio_chrdev; -+static struct class *aaudio_class; -+ -+static int aaudio_init_cmd(struct aaudio_device *a); -+static int aaudio_init_bs(struct aaudio_device *a); -+static void aaudio_init_dev(struct aaudio_device *a, aaudio_device_id_t dev_id); -+static void aaudio_free_dev(struct aaudio_subdevice *sdev); -+ -+static int aaudio_probe(struct pci_dev *dev, const struct pci_device_id *id) -+{ -+ struct aaudio_device *aaudio = NULL; -+ struct aaudio_subdevice *sdev = NULL; -+ int status = 0; -+ u32 cfg; -+ -+ pr_info("aaudio: capturing our device\n"); -+ -+ if (pci_enable_device(dev)) -+ return -ENODEV; -+ if (pci_request_regions(dev, "aaudio")) { -+ status = -ENODEV; -+ goto fail; -+ } -+ pci_set_master(dev); -+ -+ aaudio = kzalloc(sizeof(struct aaudio_device), GFP_KERNEL); -+ if (!aaudio) { -+ status = -ENOMEM; -+ goto fail; -+ } -+ -+ aaudio->bce = global_bce; -+ if (!aaudio->bce) { -+ dev_warn(&dev->dev, "aaudio: No BCE available\n"); -+ status = -EINVAL; -+ goto fail; -+ } -+ -+ aaudio->pci = dev; -+ pci_set_drvdata(dev, aaudio); -+ -+ aaudio->devt = aaudio_chrdev; -+ aaudio->dev = device_create(aaudio_class, &dev->dev, aaudio->devt, NULL, "aaudio"); -+ if (IS_ERR_OR_NULL(aaudio->dev)) { -+ status = PTR_ERR(aaudio_class); -+ goto fail; -+ } -+ device_link_add(aaudio->dev, aaudio->bce->dev, DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_CONSUMER); -+ -+ init_completion(&aaudio->remote_alive); -+ INIT_LIST_HEAD(&aaudio->subdevice_list); -+ -+ /* Init: set an unknown flag in the bitset */ -+ if (pci_read_config_dword(dev, 4, &cfg)) -+ dev_warn(&dev->dev, "aaudio: pci_read_config_dword fail\n"); -+ if (pci_write_config_dword(dev, 4, cfg | 6u)) -+ dev_warn(&dev->dev, "aaudio: pci_write_config_dword fail\n"); -+ -+ dev_info(aaudio->dev, "aaudio: bs len = %llx\n", pci_resource_len(dev, 0)); -+ aaudio->reg_mem_bs_dma = pci_resource_start(dev, 0); -+ aaudio->reg_mem_bs = pci_iomap(dev, 0, 0); -+ aaudio->reg_mem_cfg = pci_iomap(dev, 4, 0); -+ -+ aaudio->reg_mem_gpr = (u32 __iomem *) ((u8 __iomem *) aaudio->reg_mem_cfg + 0xC000); -+ -+ if (IS_ERR_OR_NULL(aaudio->reg_mem_bs) || IS_ERR_OR_NULL(aaudio->reg_mem_cfg)) { -+ dev_warn(&dev->dev, "aaudio: Failed to pci_iomap required regions\n"); -+ goto fail; -+ } -+ -+ if (aaudio_bce_init(aaudio)) { -+ dev_warn(&dev->dev, "aaudio: Failed to init BCE command transport\n"); -+ goto fail; -+ } -+ -+ if (snd_card_new(aaudio->dev, aaudio_alsa_index, aaudio_alsa_id, THIS_MODULE, 0, &aaudio->card)) { -+ dev_err(&dev->dev, "aaudio: Failed to create ALSA card\n"); -+ goto fail; -+ } -+ -+ strcpy(aaudio->card->shortname, "Apple T2 Audio"); -+ strcpy(aaudio->card->longname, "Apple T2 Audio"); -+ strcpy(aaudio->card->mixername, "Apple T2 Audio"); -+ /* Dynamic alsa ids start at 100 */ -+ aaudio->next_alsa_id = 100; -+ -+ if (aaudio_init_cmd(aaudio)) { -+ dev_err(&dev->dev, "aaudio: Failed to initialize over BCE\n"); -+ goto fail_snd; -+ } -+ -+ if (aaudio_init_bs(aaudio)) { -+ dev_err(&dev->dev, "aaudio: Failed to initialize BufferStruct\n"); -+ goto fail_snd; -+ } -+ -+ if ((status = aaudio_cmd_set_remote_access(aaudio, AAUDIO_REMOTE_ACCESS_ON))) { -+ dev_err(&dev->dev, "Failed to set remote access\n"); -+ return status; -+ } -+ -+ if (snd_card_register(aaudio->card)) { -+ dev_err(&dev->dev, "aaudio: Failed to register ALSA sound device\n"); -+ goto fail_snd; -+ } -+ -+ list_for_each_entry(sdev, &aaudio->subdevice_list, list) { -+ struct aaudio_buffer_struct_device *dev = &aaudio->bs->devices[sdev->buf_id]; -+ -+ if (sdev->out_stream_cnt == 1 && !strcmp(dev->name, "Speaker")) { -+ struct snd_pcm_hardware *hw = sdev->out_streams[0].alsa_hw_desc; -+ -+ snprintf(aaudio->card->driver, sizeof(aaudio->card->driver) / sizeof(char), "AppleT2x%d", hw->channels_min); -+ } -+ } -+ -+ return 0; -+ -+fail_snd: -+ snd_card_free(aaudio->card); -+fail: -+ if (aaudio && aaudio->dev) -+ device_destroy(aaudio_class, aaudio->devt); -+ kfree(aaudio); -+ -+ if (!IS_ERR_OR_NULL(aaudio->reg_mem_bs)) -+ pci_iounmap(dev, aaudio->reg_mem_bs); -+ if (!IS_ERR_OR_NULL(aaudio->reg_mem_cfg)) -+ pci_iounmap(dev, aaudio->reg_mem_cfg); -+ -+ pci_release_regions(dev); -+ pci_disable_device(dev); -+ -+ if (!status) -+ status = -EINVAL; -+ return status; -+} -+ -+ -+ -+static void aaudio_remove(struct pci_dev *dev) -+{ -+ struct aaudio_subdevice *sdev; -+ struct aaudio_device *aaudio = pci_get_drvdata(dev); -+ -+ snd_card_free(aaudio->card); -+ while (!list_empty(&aaudio->subdevice_list)) { -+ sdev = list_first_entry(&aaudio->subdevice_list, struct aaudio_subdevice, list); -+ list_del(&sdev->list); -+ aaudio_free_dev(sdev); -+ } -+ pci_iounmap(dev, aaudio->reg_mem_bs); -+ pci_iounmap(dev, aaudio->reg_mem_cfg); -+ device_destroy(aaudio_class, aaudio->devt); -+ pci_free_irq_vectors(dev); -+ pci_release_regions(dev); -+ pci_disable_device(dev); -+ kfree(aaudio); -+} -+ -+static int aaudio_suspend(struct device *dev) -+{ -+ struct aaudio_device *aaudio = pci_get_drvdata(to_pci_dev(dev)); -+ -+ if (aaudio_cmd_set_remote_access(aaudio, AAUDIO_REMOTE_ACCESS_OFF)) -+ dev_warn(aaudio->dev, "Failed to reset remote access\n"); -+ -+ pci_disable_device(aaudio->pci); -+ return 0; -+} -+ -+static int aaudio_resume(struct device *dev) -+{ -+ int status; -+ struct aaudio_device *aaudio = pci_get_drvdata(to_pci_dev(dev)); -+ -+ if ((status = pci_enable_device(aaudio->pci))) -+ return status; -+ pci_set_master(aaudio->pci); -+ -+ if ((status = aaudio_cmd_set_remote_access(aaudio, AAUDIO_REMOTE_ACCESS_ON))) { -+ dev_err(aaudio->dev, "Failed to set remote access\n"); -+ return status; -+ } -+ -+ return 0; -+} -+ -+static int aaudio_init_cmd(struct aaudio_device *a) -+{ -+ int status; -+ struct aaudio_send_ctx sctx; -+ struct aaudio_msg buf; -+ u64 dev_cnt, dev_i; -+ aaudio_device_id_t *dev_l; -+ -+ if ((status = aaudio_send(a, &sctx, 500, -+ aaudio_msg_write_alive_notification, 1, 3))) { -+ dev_err(a->dev, "Sending alive notification failed\n"); -+ return status; -+ } -+ -+ if (wait_for_completion_timeout(&a->remote_alive, msecs_to_jiffies(500)) == 0) { -+ dev_err(a->dev, "Timed out waiting for remote\n"); -+ return -ETIMEDOUT; -+ } -+ dev_info(a->dev, "Continuing init\n"); -+ -+ buf = aaudio_reply_alloc(); -+ if ((status = aaudio_cmd_get_device_list(a, &buf, &dev_l, &dev_cnt))) { -+ dev_err(a->dev, "Failed to get device list\n"); -+ aaudio_reply_free(&buf); -+ return status; -+ } -+ for (dev_i = 0; dev_i < dev_cnt; ++dev_i) -+ aaudio_init_dev(a, dev_l[dev_i]); -+ aaudio_reply_free(&buf); -+ -+ return 0; -+} -+ -+static void aaudio_init_stream_info(struct aaudio_subdevice *sdev, struct aaudio_stream *strm); -+static void aaudio_handle_jack_connection_change(struct aaudio_subdevice *sdev); -+ -+static void aaudio_init_dev(struct aaudio_device *a, aaudio_device_id_t dev_id) -+{ -+ struct aaudio_subdevice *sdev; -+ struct aaudio_msg buf = aaudio_reply_alloc(); -+ u64 uid_len, stream_cnt, i; -+ aaudio_object_id_t *stream_list; -+ char *uid; -+ -+ sdev = kzalloc(sizeof(struct aaudio_subdevice), GFP_KERNEL); -+ -+ if (aaudio_cmd_get_property(a, &buf, dev_id, dev_id, AAUDIO_PROP(AAUDIO_PROP_SCOPE_GLOBAL, AAUDIO_PROP_UID, 0), -+ NULL, 0, (void **) &uid, &uid_len) || uid_len > AAUDIO_DEVICE_MAX_UID_LEN) { -+ dev_err(a->dev, "Failed to get device uid for device %llx\n", dev_id); -+ goto fail; -+ } -+ dev_info(a->dev, "Remote device %llx %.*s\n", dev_id, (int) uid_len, uid); -+ -+ sdev->a = a; -+ INIT_LIST_HEAD(&sdev->list); -+ sdev->dev_id = dev_id; -+ sdev->buf_id = AAUDIO_BUFFER_ID_NONE; -+ strncpy(sdev->uid, uid, uid_len); -+ sdev->uid[uid_len + 1] = '\0'; -+ -+ if (aaudio_cmd_get_primitive_property(a, dev_id, dev_id, -+ AAUDIO_PROP(AAUDIO_PROP_SCOPE_INPUT, AAUDIO_PROP_LATENCY, 0), NULL, 0, &sdev->in_latency, sizeof(u32))) -+ dev_warn(a->dev, "Failed to query device input latency\n"); -+ if (aaudio_cmd_get_primitive_property(a, dev_id, dev_id, -+ AAUDIO_PROP(AAUDIO_PROP_SCOPE_OUTPUT, AAUDIO_PROP_LATENCY, 0), NULL, 0, &sdev->out_latency, sizeof(u32))) -+ dev_warn(a->dev, "Failed to query device output latency\n"); -+ -+ if (aaudio_cmd_get_input_stream_list(a, &buf, dev_id, &stream_list, &stream_cnt)) { -+ dev_err(a->dev, "Failed to get input stream list for device %llx\n", dev_id); -+ goto fail; -+ } -+ if (stream_cnt > AAUDIO_DEIVCE_MAX_INPUT_STREAMS) { -+ dev_warn(a->dev, "Device %s input stream count %llu is larger than the supported count of %u\n", -+ sdev->uid, stream_cnt, AAUDIO_DEIVCE_MAX_INPUT_STREAMS); -+ stream_cnt = AAUDIO_DEIVCE_MAX_INPUT_STREAMS; -+ } -+ sdev->in_stream_cnt = stream_cnt; -+ for (i = 0; i < stream_cnt; i++) { -+ sdev->in_streams[i].id = stream_list[i]; -+ sdev->in_streams[i].buffer_cnt = 0; -+ aaudio_init_stream_info(sdev, &sdev->in_streams[i]); -+ sdev->in_streams[i].latency += sdev->in_latency; -+ } -+ -+ if (aaudio_cmd_get_output_stream_list(a, &buf, dev_id, &stream_list, &stream_cnt)) { -+ dev_err(a->dev, "Failed to get output stream list for device %llx\n", dev_id); -+ goto fail; -+ } -+ if (stream_cnt > AAUDIO_DEIVCE_MAX_OUTPUT_STREAMS) { -+ dev_warn(a->dev, "Device %s input stream count %llu is larger than the supported count of %u\n", -+ sdev->uid, stream_cnt, AAUDIO_DEIVCE_MAX_OUTPUT_STREAMS); -+ stream_cnt = AAUDIO_DEIVCE_MAX_OUTPUT_STREAMS; -+ } -+ sdev->out_stream_cnt = stream_cnt; -+ for (i = 0; i < stream_cnt; i++) { -+ sdev->out_streams[i].id = stream_list[i]; -+ sdev->out_streams[i].buffer_cnt = 0; -+ aaudio_init_stream_info(sdev, &sdev->out_streams[i]); -+ sdev->out_streams[i].latency += sdev->in_latency; -+ } -+ -+ if (sdev->is_pcm) -+ aaudio_create_pcm(sdev); -+ /* Headphone Jack status */ -+ if (!strcmp(sdev->uid, "Codec Output")) { -+ if (snd_jack_new(a->card, sdev->uid, SND_JACK_HEADPHONE, &sdev->jack, true, false)) -+ dev_warn(a->dev, "Failed to create an attached jack for %s\n", sdev->uid); -+ aaudio_cmd_property_listener(a, sdev->dev_id, sdev->dev_id, -+ AAUDIO_PROP(AAUDIO_PROP_SCOPE_OUTPUT, AAUDIO_PROP_JACK_PLUGGED, 0)); -+ aaudio_handle_jack_connection_change(sdev); -+ } -+ -+ aaudio_reply_free(&buf); -+ -+ list_add_tail(&sdev->list, &a->subdevice_list); -+ return; -+ -+fail: -+ aaudio_reply_free(&buf); -+ kfree(sdev); -+} -+ -+static void aaudio_init_stream_info(struct aaudio_subdevice *sdev, struct aaudio_stream *strm) -+{ -+ if (aaudio_cmd_get_primitive_property(sdev->a, sdev->dev_id, strm->id, -+ AAUDIO_PROP(AAUDIO_PROP_SCOPE_GLOBAL, AAUDIO_PROP_PHYS_FORMAT, 0), NULL, 0, -+ &strm->desc, sizeof(strm->desc))) -+ dev_warn(sdev->a->dev, "Failed to query stream descriptor\n"); -+ if (aaudio_cmd_get_primitive_property(sdev->a, sdev->dev_id, strm->id, -+ AAUDIO_PROP(AAUDIO_PROP_SCOPE_GLOBAL, AAUDIO_PROP_LATENCY, 0), NULL, 0, &strm->latency, sizeof(u32))) -+ dev_warn(sdev->a->dev, "Failed to query stream latency\n"); -+ if (strm->desc.format_id == AAUDIO_FORMAT_LPCM) -+ sdev->is_pcm = true; -+} -+ -+static void aaudio_free_dev(struct aaudio_subdevice *sdev) -+{ -+ size_t i; -+ for (i = 0; i < sdev->in_stream_cnt; i++) { -+ if (sdev->in_streams[i].alsa_hw_desc) -+ kfree(sdev->in_streams[i].alsa_hw_desc); -+ if (sdev->in_streams[i].buffers) -+ kfree(sdev->in_streams[i].buffers); -+ } -+ for (i = 0; i < sdev->out_stream_cnt; i++) { -+ if (sdev->out_streams[i].alsa_hw_desc) -+ kfree(sdev->out_streams[i].alsa_hw_desc); -+ if (sdev->out_streams[i].buffers) -+ kfree(sdev->out_streams[i].buffers); -+ } -+ kfree(sdev); -+} -+ -+static struct aaudio_subdevice *aaudio_find_dev_by_dev_id(struct aaudio_device *a, aaudio_device_id_t dev_id) -+{ -+ struct aaudio_subdevice *sdev; -+ list_for_each_entry(sdev, &a->subdevice_list, list) { -+ if (dev_id == sdev->dev_id) -+ return sdev; -+ } -+ return NULL; -+} -+ -+static struct aaudio_subdevice *aaudio_find_dev_by_uid(struct aaudio_device *a, const char *uid) -+{ -+ struct aaudio_subdevice *sdev; -+ list_for_each_entry(sdev, &a->subdevice_list, list) { -+ if (!strcmp(uid, sdev->uid)) -+ return sdev; -+ } -+ return NULL; -+} -+ -+static void aaudio_init_bs_stream(struct aaudio_device *a, struct aaudio_stream *strm, -+ struct aaudio_buffer_struct_stream *bs_strm); -+static void aaudio_init_bs_stream_host(struct aaudio_device *a, struct aaudio_stream *strm, -+ struct aaudio_buffer_struct_stream *bs_strm); -+ -+static int aaudio_init_bs(struct aaudio_device *a) -+{ -+ int i, j; -+ struct aaudio_buffer_struct_device *dev; -+ struct aaudio_subdevice *sdev; -+ u32 ver, sig, bs_base; -+ -+ ver = ioread32(&a->reg_mem_gpr[0]); -+ if (ver < 3) { -+ dev_err(a->dev, "aaudio: Bad GPR version (%u)", ver); -+ return -EINVAL; -+ } -+ sig = ioread32(&a->reg_mem_gpr[1]); -+ if (sig != AAUDIO_SIG) { -+ dev_err(a->dev, "aaudio: Bad GPR sig (%x)", sig); -+ return -EINVAL; -+ } -+ bs_base = ioread32(&a->reg_mem_gpr[2]); -+ a->bs = (struct aaudio_buffer_struct *) ((u8 *) a->reg_mem_bs + bs_base); -+ if (a->bs->signature != AAUDIO_SIG) { -+ dev_err(a->dev, "aaudio: Bad BufferStruct sig (%x)", a->bs->signature); -+ return -EINVAL; -+ } -+ dev_info(a->dev, "aaudio: BufferStruct ver = %i\n", a->bs->version); -+ dev_info(a->dev, "aaudio: Num devices = %i\n", a->bs->num_devices); -+ for (i = 0; i < a->bs->num_devices; i++) { -+ dev = &a->bs->devices[i]; -+ dev_info(a->dev, "aaudio: Device %i %s\n", i, dev->name); -+ -+ sdev = aaudio_find_dev_by_uid(a, dev->name); -+ if (!sdev) { -+ dev_err(a->dev, "aaudio: Subdevice not found for BufferStruct device %s\n", dev->name); -+ continue; -+ } -+ sdev->buf_id = (u8) i; -+ dev->num_input_streams = 0; -+ for (j = 0; j < dev->num_output_streams; j++) { -+ dev_info(a->dev, "aaudio: Device %i Stream %i: Output; Buffer Count = %i\n", i, j, -+ dev->output_streams[j].num_buffers); -+ if (j < sdev->out_stream_cnt) -+ aaudio_init_bs_stream(a, &sdev->out_streams[j], &dev->output_streams[j]); -+ } -+ } -+ -+ list_for_each_entry(sdev, &a->subdevice_list, list) { -+ if (sdev->buf_id != AAUDIO_BUFFER_ID_NONE) -+ continue; -+ sdev->buf_id = i; -+ dev_info(a->dev, "aaudio: Created device %i %s\n", i, sdev->uid); -+ strcpy(a->bs->devices[i].name, sdev->uid); -+ a->bs->devices[i].num_input_streams = 0; -+ a->bs->devices[i].num_output_streams = 0; -+ a->bs->num_devices = ++i; -+ } -+ list_for_each_entry(sdev, &a->subdevice_list, list) { -+ if (sdev->in_stream_cnt == 1) { -+ dev_info(a->dev, "aaudio: Device %i Host Stream; Input\n", sdev->buf_id); -+ aaudio_init_bs_stream_host(a, &sdev->in_streams[0], &a->bs->devices[sdev->buf_id].input_streams[0]); -+ a->bs->devices[sdev->buf_id].num_input_streams = 1; -+ wmb(); -+ -+ if (aaudio_cmd_set_input_stream_address_ranges(a, sdev->dev_id)) { -+ dev_err(a->dev, "aaudio: Failed to set input stream address ranges\n"); -+ } -+ } -+ } -+ -+ return 0; -+} -+ -+static void aaudio_init_bs_stream(struct aaudio_device *a, struct aaudio_stream *strm, -+ struct aaudio_buffer_struct_stream *bs_strm) -+{ -+ size_t i; -+ strm->buffer_cnt = bs_strm->num_buffers; -+ if (bs_strm->num_buffers > AAUDIO_DEIVCE_MAX_BUFFER_COUNT) { -+ dev_warn(a->dev, "BufferStruct buffer count %u exceeds driver limit of %u\n", bs_strm->num_buffers, -+ AAUDIO_DEIVCE_MAX_BUFFER_COUNT); -+ strm->buffer_cnt = AAUDIO_DEIVCE_MAX_BUFFER_COUNT; -+ } -+ if (!strm->buffer_cnt) -+ return; -+ strm->buffers = kmalloc_array(strm->buffer_cnt, sizeof(struct aaudio_dma_buf), GFP_KERNEL); -+ if (!strm->buffers) { -+ dev_err(a->dev, "Buffer list allocation failed\n"); -+ return; -+ } -+ for (i = 0; i < strm->buffer_cnt; i++) { -+ strm->buffers[i].dma_addr = a->reg_mem_bs_dma + (dma_addr_t) bs_strm->buffers[i].address; -+ strm->buffers[i].ptr = a->reg_mem_bs + bs_strm->buffers[i].address; -+ strm->buffers[i].size = bs_strm->buffers[i].size; -+ } -+ -+ if (strm->buffer_cnt == 1) { -+ strm->alsa_hw_desc = kmalloc(sizeof(struct snd_pcm_hardware), GFP_KERNEL); -+ if (aaudio_create_hw_info(&strm->desc, strm->alsa_hw_desc, strm->buffers[0].size)) { -+ kfree(strm->alsa_hw_desc); -+ strm->alsa_hw_desc = NULL; -+ } -+ } -+} -+ -+static void aaudio_init_bs_stream_host(struct aaudio_device *a, struct aaudio_stream *strm, -+ struct aaudio_buffer_struct_stream *bs_strm) -+{ -+ size_t size; -+ dma_addr_t dma_addr; -+ void *dma_ptr; -+ size = strm->desc.bytes_per_packet * 16640; -+ dma_ptr = dma_alloc_coherent(&a->pci->dev, size, &dma_addr, GFP_KERNEL); -+ if (!dma_ptr) { -+ dev_err(a->dev, "dma_alloc_coherent failed\n"); -+ return; -+ } -+ bs_strm->buffers[0].address = dma_addr; -+ bs_strm->buffers[0].size = size; -+ bs_strm->num_buffers = 1; -+ -+ memset(dma_ptr, 0, size); -+ -+ strm->buffer_cnt = 1; -+ strm->buffers = kmalloc_array(strm->buffer_cnt, sizeof(struct aaudio_dma_buf), GFP_KERNEL); -+ if (!strm->buffers) { -+ dev_err(a->dev, "Buffer list allocation failed\n"); -+ return; -+ } -+ strm->buffers[0].dma_addr = dma_addr; -+ strm->buffers[0].ptr = dma_ptr; -+ strm->buffers[0].size = size; -+ -+ strm->alsa_hw_desc = kmalloc(sizeof(struct snd_pcm_hardware), GFP_KERNEL); -+ if (aaudio_create_hw_info(&strm->desc, strm->alsa_hw_desc, strm->buffers[0].size)) { -+ kfree(strm->alsa_hw_desc); -+ strm->alsa_hw_desc = NULL; -+ } -+} -+ -+static void aaudio_handle_prop_change(struct aaudio_device *a, struct aaudio_msg *msg); -+ -+void aaudio_handle_notification(struct aaudio_device *a, struct aaudio_msg *msg) -+{ -+ struct aaudio_send_ctx sctx; -+ struct aaudio_msg_base base; -+ if (aaudio_msg_read_base(msg, &base)) -+ return; -+ switch (base.msg) { -+ case AAUDIO_MSG_NOTIFICATION_BOOT: -+ dev_info(a->dev, "Received boot notification from remote\n"); -+ -+ /* Resend the alive notify */ -+ if (aaudio_send(a, &sctx, 500, -+ aaudio_msg_write_alive_notification, 1, 3)) { -+ pr_err("Sending alive notification failed\n"); -+ } -+ break; -+ case AAUDIO_MSG_NOTIFICATION_ALIVE: -+ dev_info(a->dev, "Received alive notification from remote\n"); -+ complete_all(&a->remote_alive); -+ break; -+ case AAUDIO_MSG_PROPERTY_CHANGED: -+ aaudio_handle_prop_change(a, msg); -+ break; -+ default: -+ dev_info(a->dev, "Unhandled notification %i", base.msg); -+ break; -+ } -+} -+ -+struct aaudio_prop_change_work_struct { -+ struct work_struct ws; -+ struct aaudio_device *a; -+ aaudio_device_id_t dev; -+ aaudio_object_id_t obj; -+ struct aaudio_prop_addr prop; -+}; -+ -+static void aaudio_handle_jack_connection_change(struct aaudio_subdevice *sdev) -+{ -+ u32 plugged; -+ if (!sdev->jack) -+ return; -+ /* NOTE: Apple made the plug status scoped to the input and output streams. This makes no sense for us, so I just -+ * always pick the OUTPUT status. */ -+ if (aaudio_cmd_get_primitive_property(sdev->a, sdev->dev_id, sdev->dev_id, -+ AAUDIO_PROP(AAUDIO_PROP_SCOPE_OUTPUT, AAUDIO_PROP_JACK_PLUGGED, 0), NULL, 0, &plugged, sizeof(plugged))) { -+ dev_err(sdev->a->dev, "Failed to get jack enable status\n"); -+ return; -+ } -+ dev_dbg(sdev->a->dev, "Jack is now %s\n", plugged ? "plugged" : "unplugged"); -+ snd_jack_report(sdev->jack, plugged ? sdev->jack->type : 0); -+} -+ -+void aaudio_handle_prop_change_work(struct work_struct *ws) -+{ -+ struct aaudio_prop_change_work_struct *work = container_of(ws, struct aaudio_prop_change_work_struct, ws); -+ struct aaudio_subdevice *sdev; -+ -+ sdev = aaudio_find_dev_by_dev_id(work->a, work->dev); -+ if (!sdev) { -+ dev_err(work->a->dev, "Property notification change: device not found\n"); -+ goto done; -+ } -+ dev_dbg(work->a->dev, "Property changed for device: %s\n", sdev->uid); -+ -+ if (work->prop.scope == AAUDIO_PROP_SCOPE_OUTPUT && work->prop.selector == AAUDIO_PROP_JACK_PLUGGED) { -+ aaudio_handle_jack_connection_change(sdev); -+ } -+ -+done: -+ kfree(work); -+} -+ -+void aaudio_handle_prop_change(struct aaudio_device *a, struct aaudio_msg *msg) -+{ -+ /* NOTE: This is a scheduled work because this callback will generally need to query device information and this -+ * is not possible when we are in the reply parsing code's context. */ -+ struct aaudio_prop_change_work_struct *work; -+ work = kmalloc(sizeof(struct aaudio_prop_change_work_struct), GFP_KERNEL); -+ work->a = a; -+ INIT_WORK(&work->ws, aaudio_handle_prop_change_work); -+ aaudio_msg_read_property_changed(msg, &work->dev, &work->obj, &work->prop); -+ schedule_work(&work->ws); -+} -+ -+#define aaudio_send_cmd_response(a, sctx, msg, fn, ...) \ -+ if (aaudio_send_with_tag(a, sctx, ((struct aaudio_msg_header *) msg->data)->tag, 500, fn, ##__VA_ARGS__)) \ -+ pr_err("aaudio: Failed to reply to a command\n"); -+ -+void aaudio_handle_cmd_timestamp(struct aaudio_device *a, struct aaudio_msg *msg) -+{ -+ ktime_t time_os = ktime_get_boottime(); -+ struct aaudio_send_ctx sctx; -+ struct aaudio_subdevice *sdev; -+ u64 devid, timestamp, update_seed; -+ aaudio_msg_read_update_timestamp(msg, &devid, ×tamp, &update_seed); -+ dev_dbg(a->dev, "Received timestamp update for dev=%llx ts=%llx seed=%llx\n", devid, timestamp, update_seed); -+ -+ sdev = aaudio_find_dev_by_dev_id(a, devid); -+ aaudio_handle_timestamp(sdev, time_os, timestamp); -+ -+ aaudio_send_cmd_response(a, &sctx, msg, -+ aaudio_msg_write_update_timestamp_response); -+} -+ -+void aaudio_handle_command(struct aaudio_device *a, struct aaudio_msg *msg) -+{ -+ struct aaudio_msg_base base; -+ if (aaudio_msg_read_base(msg, &base)) -+ return; -+ switch (base.msg) { -+ case AAUDIO_MSG_UPDATE_TIMESTAMP: -+ aaudio_handle_cmd_timestamp(a, msg); -+ break; -+ default: -+ dev_info(a->dev, "Unhandled device command %i", base.msg); -+ break; -+ } -+} -+ -+static struct pci_device_id aaudio_ids[ ] = { -+ { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1803) }, -+ { 0, }, -+}; -+ -+struct dev_pm_ops aaudio_pci_driver_pm = { -+ .suspend = aaudio_suspend, -+ .resume = aaudio_resume -+}; -+struct pci_driver aaudio_pci_driver = { -+ .name = "aaudio", -+ .id_table = aaudio_ids, -+ .probe = aaudio_probe, -+ .remove = aaudio_remove, -+ .driver = { -+ .pm = &aaudio_pci_driver_pm -+ } -+}; -+ -+ -+int aaudio_module_init(void) -+{ -+ int result; -+ if ((result = alloc_chrdev_region(&aaudio_chrdev, 0, 1, "aaudio"))) -+ goto fail_chrdev; -+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,4,0) -+ aaudio_class = class_create(THIS_MODULE, "aaudio"); -+#else -+ aaudio_class = class_create("aaudio"); -+#endif -+ if (IS_ERR(aaudio_class)) { -+ result = PTR_ERR(aaudio_class); -+ goto fail_class; -+ } -+ -+ result = pci_register_driver(&aaudio_pci_driver); -+ if (result) -+ goto fail_drv; -+ return 0; -+ -+fail_drv: -+ pci_unregister_driver(&aaudio_pci_driver); -+fail_class: -+ class_destroy(aaudio_class); -+fail_chrdev: -+ unregister_chrdev_region(aaudio_chrdev, 1); -+ if (!result) -+ result = -EINVAL; -+ return result; -+} -+ -+void aaudio_module_exit(void) -+{ -+ pci_unregister_driver(&aaudio_pci_driver); -+ class_destroy(aaudio_class); -+ unregister_chrdev_region(aaudio_chrdev, 1); -+} -+ -+struct aaudio_alsa_pcm_id_mapping aaudio_alsa_id_mappings[] = { -+ {"Speaker", 0}, -+ {"Digital Mic", 1}, -+ {"Codec Output", 2}, -+ {"Codec Input", 3}, -+ {"Bridge Loopback", 4}, -+ {} -+}; -+ -+module_param_named(index, aaudio_alsa_index, int, 0444); -+MODULE_PARM_DESC(index, "Index value for Apple Internal Audio soundcard."); -+module_param_named(id, aaudio_alsa_id, charp, 0444); -+MODULE_PARM_DESC(id, "ID string for Apple Internal Audio soundcard."); -diff --git a/drivers/staging/apple-bce/audio/audio.h b/drivers/staging/apple-bce/audio/audio.h -new file mode 100644 -index 000000000000..004bc1e22ea4 ---- /dev/null -+++ b/drivers/staging/apple-bce/audio/audio.h -@@ -0,0 +1,125 @@ -+#ifndef AAUDIO_H -+#define AAUDIO_H -+ -+#include -+#include -+#include "../apple_bce.h" -+#include "protocol_bce.h" -+#include "description.h" -+ -+#define AAUDIO_SIG 0x19870423 -+ -+#define AAUDIO_DEVICE_MAX_UID_LEN 128 -+#define AAUDIO_DEIVCE_MAX_INPUT_STREAMS 1 -+#define AAUDIO_DEIVCE_MAX_OUTPUT_STREAMS 1 -+#define AAUDIO_DEIVCE_MAX_BUFFER_COUNT 1 -+ -+#define AAUDIO_BUFFER_ID_NONE 0xffu -+ -+struct snd_card; -+struct snd_pcm; -+struct snd_pcm_hardware; -+struct snd_jack; -+ -+struct __attribute__((packed)) __attribute__((aligned(4))) aaudio_buffer_struct_buffer { -+ size_t address; -+ size_t size; -+ size_t pad[4]; -+}; -+struct aaudio_buffer_struct_stream { -+ u8 num_buffers; -+ struct aaudio_buffer_struct_buffer buffers[100]; -+ char filler[32]; -+}; -+struct aaudio_buffer_struct_device { -+ char name[128]; -+ u8 num_input_streams; -+ u8 num_output_streams; -+ struct aaudio_buffer_struct_stream input_streams[5]; -+ struct aaudio_buffer_struct_stream output_streams[5]; -+ char filler[128]; -+}; -+struct aaudio_buffer_struct { -+ u32 version; -+ u32 signature; -+ u32 flags; -+ u8 num_devices; -+ struct aaudio_buffer_struct_device devices[20]; -+}; -+ -+struct aaudio_device; -+struct aaudio_dma_buf { -+ dma_addr_t dma_addr; -+ void *ptr; -+ size_t size; -+}; -+struct aaudio_stream { -+ aaudio_object_id_t id; -+ size_t buffer_cnt; -+ struct aaudio_dma_buf *buffers; -+ -+ struct aaudio_apple_description desc; -+ struct snd_pcm_hardware *alsa_hw_desc; -+ u32 latency; -+ -+ bool waiting_for_first_ts; -+ -+ ktime_t remote_timestamp; -+ snd_pcm_sframes_t frame_min; -+ int started; -+}; -+struct aaudio_subdevice { -+ struct aaudio_device *a; -+ struct list_head list; -+ aaudio_device_id_t dev_id; -+ u32 in_latency, out_latency; -+ u8 buf_id; -+ int alsa_id; -+ char uid[AAUDIO_DEVICE_MAX_UID_LEN + 1]; -+ size_t in_stream_cnt; -+ struct aaudio_stream in_streams[AAUDIO_DEIVCE_MAX_INPUT_STREAMS]; -+ size_t out_stream_cnt; -+ struct aaudio_stream out_streams[AAUDIO_DEIVCE_MAX_OUTPUT_STREAMS]; -+ bool is_pcm; -+ struct snd_pcm *pcm; -+ struct snd_jack *jack; -+}; -+struct aaudio_alsa_pcm_id_mapping { -+ const char *name; -+ int alsa_id; -+}; -+ -+struct aaudio_device { -+ struct pci_dev *pci; -+ dev_t devt; -+ struct device *dev; -+ void __iomem *reg_mem_bs; -+ dma_addr_t reg_mem_bs_dma; -+ void __iomem *reg_mem_cfg; -+ -+ u32 __iomem *reg_mem_gpr; -+ -+ struct aaudio_buffer_struct *bs; -+ -+ struct apple_bce_device *bce; -+ struct aaudio_bce bcem; -+ -+ struct snd_card *card; -+ -+ struct list_head subdevice_list; -+ int next_alsa_id; -+ -+ struct completion remote_alive; -+}; -+ -+void aaudio_handle_notification(struct aaudio_device *a, struct aaudio_msg *msg); -+void aaudio_handle_prop_change_work(struct work_struct *ws); -+void aaudio_handle_cmd_timestamp(struct aaudio_device *a, struct aaudio_msg *msg); -+void aaudio_handle_command(struct aaudio_device *a, struct aaudio_msg *msg); -+ -+int aaudio_module_init(void); -+void aaudio_module_exit(void); -+ -+extern struct aaudio_alsa_pcm_id_mapping aaudio_alsa_id_mappings[]; -+ -+#endif //AAUDIO_H -diff --git a/drivers/staging/apple-bce/audio/description.h b/drivers/staging/apple-bce/audio/description.h -new file mode 100644 -index 000000000000..dfef3ab68f27 ---- /dev/null -+++ b/drivers/staging/apple-bce/audio/description.h -@@ -0,0 +1,42 @@ -+#ifndef AAUDIO_DESCRIPTION_H -+#define AAUDIO_DESCRIPTION_H -+ -+#include -+ -+struct aaudio_apple_description { -+ u64 sample_rate_double; -+ u32 format_id; -+ u32 format_flags; -+ u32 bytes_per_packet; -+ u32 frames_per_packet; -+ u32 bytes_per_frame; -+ u32 channels_per_frame; -+ u32 bits_per_channel; -+ u32 reserved; -+}; -+ -+enum { -+ AAUDIO_FORMAT_LPCM = 0x6c70636d // 'lpcm' -+}; -+ -+enum { -+ AAUDIO_FORMAT_FLAG_FLOAT = 1, -+ AAUDIO_FORMAT_FLAG_BIG_ENDIAN = 2, -+ AAUDIO_FORMAT_FLAG_SIGNED = 4, -+ AAUDIO_FORMAT_FLAG_PACKED = 8, -+ AAUDIO_FORMAT_FLAG_ALIGNED_HIGH = 16, -+ AAUDIO_FORMAT_FLAG_NON_INTERLEAVED = 32, -+ AAUDIO_FORMAT_FLAG_NON_MIXABLE = 64 -+}; -+ -+static inline u64 aaudio_double_to_u64(u64 d) -+{ -+ u8 sign = (u8) ((d >> 63) & 1); -+ s32 exp = (s32) ((d >> 52) & 0x7ff) - 1023; -+ u64 fr = d & ((1LL << 52) - 1); -+ if (sign || exp < 0) -+ return 0; -+ return (u64) ((1LL << exp) + (fr >> (52 - exp))); -+} -+ -+#endif //AAUDIO_DESCRIPTION_H -diff --git a/drivers/staging/apple-bce/audio/pcm.c b/drivers/staging/apple-bce/audio/pcm.c -new file mode 100644 -index 000000000000..1026e10a9ac5 ---- /dev/null -+++ b/drivers/staging/apple-bce/audio/pcm.c -@@ -0,0 +1,308 @@ -+#include "pcm.h" -+#include "audio.h" -+ -+static u64 aaudio_get_alsa_fmtbit(struct aaudio_apple_description *desc) -+{ -+ if (desc->format_flags & AAUDIO_FORMAT_FLAG_FLOAT) { -+ if (desc->bits_per_channel == 32) { -+ if (desc->format_flags & AAUDIO_FORMAT_FLAG_BIG_ENDIAN) -+ return SNDRV_PCM_FMTBIT_FLOAT_BE; -+ else -+ return SNDRV_PCM_FMTBIT_FLOAT_LE; -+ } else if (desc->bits_per_channel == 64) { -+ if (desc->format_flags & AAUDIO_FORMAT_FLAG_BIG_ENDIAN) -+ return SNDRV_PCM_FMTBIT_FLOAT64_BE; -+ else -+ return SNDRV_PCM_FMTBIT_FLOAT64_LE; -+ } else { -+ pr_err("aaudio: unsupported bits per channel for float format: %u\n", desc->bits_per_channel); -+ return 0; -+ } -+ } -+#define DEFINE_BPC_OPTION(val, b) \ -+ case val: \ -+ if (desc->format_flags & AAUDIO_FORMAT_FLAG_BIG_ENDIAN) { \ -+ if (desc->format_flags & AAUDIO_FORMAT_FLAG_SIGNED) \ -+ return SNDRV_PCM_FMTBIT_S ## b ## BE; \ -+ else \ -+ return SNDRV_PCM_FMTBIT_U ## b ## BE; \ -+ } else { \ -+ if (desc->format_flags & AAUDIO_FORMAT_FLAG_SIGNED) \ -+ return SNDRV_PCM_FMTBIT_S ## b ## LE; \ -+ else \ -+ return SNDRV_PCM_FMTBIT_U ## b ## LE; \ -+ } -+ if (desc->format_flags & AAUDIO_FORMAT_FLAG_PACKED) { -+ switch (desc->bits_per_channel) { -+ case 8: -+ case 16: -+ case 32: -+ break; -+ DEFINE_BPC_OPTION(24, 24_3) -+ default: -+ pr_err("aaudio: unsupported bits per channel for packed format: %u\n", desc->bits_per_channel); -+ return 0; -+ } -+ } -+ if (desc->format_flags & AAUDIO_FORMAT_FLAG_ALIGNED_HIGH) { -+ switch (desc->bits_per_channel) { -+ DEFINE_BPC_OPTION(24, 32_) -+ default: -+ pr_err("aaudio: unsupported bits per channel for high-aligned format: %u\n", desc->bits_per_channel); -+ return 0; -+ } -+ } -+ switch (desc->bits_per_channel) { -+ case 8: -+ if (desc->format_flags & AAUDIO_FORMAT_FLAG_SIGNED) -+ return SNDRV_PCM_FMTBIT_S8; -+ else -+ return SNDRV_PCM_FMTBIT_U8; -+ DEFINE_BPC_OPTION(16, 16_) -+ DEFINE_BPC_OPTION(24, 24_) -+ DEFINE_BPC_OPTION(32, 32_) -+ default: -+ pr_err("aaudio: unsupported bits per channel: %u\n", desc->bits_per_channel); -+ return 0; -+ } -+} -+int aaudio_create_hw_info(struct aaudio_apple_description *desc, struct snd_pcm_hardware *alsa_hw, -+ size_t buf_size) -+{ -+ uint rate; -+ alsa_hw->info = (SNDRV_PCM_INFO_MMAP | -+ SNDRV_PCM_INFO_BLOCK_TRANSFER | -+ SNDRV_PCM_INFO_MMAP_VALID | -+ SNDRV_PCM_INFO_DOUBLE); -+ if (desc->format_flags & AAUDIO_FORMAT_FLAG_NON_MIXABLE) -+ pr_warn("aaudio: unsupported hw flag: NON_MIXABLE\n"); -+ if (!(desc->format_flags & AAUDIO_FORMAT_FLAG_NON_INTERLEAVED)) -+ alsa_hw->info |= SNDRV_PCM_INFO_INTERLEAVED; -+ alsa_hw->formats = aaudio_get_alsa_fmtbit(desc); -+ if (!alsa_hw->formats) -+ return -EINVAL; -+ rate = (uint) aaudio_double_to_u64(desc->sample_rate_double); -+ alsa_hw->rates = snd_pcm_rate_to_rate_bit(rate); -+ alsa_hw->rate_min = rate; -+ alsa_hw->rate_max = rate; -+ alsa_hw->channels_min = desc->channels_per_frame; -+ alsa_hw->channels_max = desc->channels_per_frame; -+ alsa_hw->buffer_bytes_max = buf_size; -+ alsa_hw->period_bytes_min = desc->bytes_per_packet; -+ alsa_hw->period_bytes_max = desc->bytes_per_packet; -+ alsa_hw->periods_min = (uint) (buf_size / desc->bytes_per_packet); -+ alsa_hw->periods_max = (uint) (buf_size / desc->bytes_per_packet); -+ pr_debug("aaudio_create_hw_info: format = %llu, rate = %u/%u. channels = %u, periods = %u, period size = %lu\n", -+ alsa_hw->formats, alsa_hw->rate_min, alsa_hw->rates, alsa_hw->channels_min, alsa_hw->periods_min, -+ alsa_hw->period_bytes_min); -+ return 0; -+} -+ -+static struct aaudio_stream *aaudio_pcm_stream(struct snd_pcm_substream *substream) -+{ -+ struct aaudio_subdevice *sdev = snd_pcm_substream_chip(substream); -+ if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) -+ return &sdev->out_streams[substream->number]; -+ else -+ return &sdev->in_streams[substream->number]; -+} -+ -+static int aaudio_pcm_open(struct snd_pcm_substream *substream) -+{ -+ pr_debug("aaudio_pcm_open\n"); -+ substream->runtime->hw = *aaudio_pcm_stream(substream)->alsa_hw_desc; -+ -+ return 0; -+} -+ -+static int aaudio_pcm_close(struct snd_pcm_substream *substream) -+{ -+ pr_debug("aaudio_pcm_close\n"); -+ return 0; -+} -+ -+static int aaudio_pcm_prepare(struct snd_pcm_substream *substream) -+{ -+ return 0; -+} -+ -+static int aaudio_pcm_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *hw_params) -+{ -+ struct aaudio_stream *astream = aaudio_pcm_stream(substream); -+ pr_debug("aaudio_pcm_hw_params\n"); -+ -+ if (!astream->buffer_cnt || !astream->buffers) -+ return -EINVAL; -+ -+ substream->runtime->dma_area = astream->buffers[0].ptr; -+ substream->runtime->dma_addr = astream->buffers[0].dma_addr; -+ substream->runtime->dma_bytes = astream->buffers[0].size; -+ return 0; -+} -+ -+static int aaudio_pcm_hw_free(struct snd_pcm_substream *substream) -+{ -+ pr_debug("aaudio_pcm_hw_free\n"); -+ return 0; -+} -+ -+static void aaudio_pcm_start(struct snd_pcm_substream *substream) -+{ -+ struct aaudio_subdevice *sdev = snd_pcm_substream_chip(substream); -+ struct aaudio_stream *stream = aaudio_pcm_stream(substream); -+ void *buf; -+ size_t s; -+ ktime_t time_start, time_end; -+ bool back_buffer; -+ time_start = ktime_get(); -+ -+ back_buffer = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK); -+ -+ if (back_buffer) { -+ s = frames_to_bytes(substream->runtime, substream->runtime->control->appl_ptr); -+ buf = kmalloc(s, GFP_KERNEL); -+ memcpy_fromio(buf, substream->runtime->dma_area, s); -+ time_end = ktime_get(); -+ pr_debug("aaudio: Backed up the buffer in %lluns [%li]\n", ktime_to_ns(time_end - time_start), -+ substream->runtime->control->appl_ptr); -+ } -+ -+ stream->waiting_for_first_ts = true; -+ stream->frame_min = stream->latency; -+ -+ aaudio_cmd_start_io(sdev->a, sdev->dev_id); -+ if (back_buffer) -+ memcpy_toio(substream->runtime->dma_area, buf, s); -+ -+ time_end = ktime_get(); -+ pr_debug("aaudio: Started the audio device in %lluns\n", ktime_to_ns(time_end - time_start)); -+} -+ -+static int aaudio_pcm_trigger(struct snd_pcm_substream *substream, int cmd) -+{ -+ struct aaudio_subdevice *sdev = snd_pcm_substream_chip(substream); -+ struct aaudio_stream *stream = aaudio_pcm_stream(substream); -+ pr_debug("aaudio_pcm_trigger %x\n", cmd); -+ -+ /* We only supports triggers on the #0 buffer */ -+ if (substream->number != 0) -+ return 0; -+ switch (cmd) { -+ case SNDRV_PCM_TRIGGER_START: -+ aaudio_pcm_start(substream); -+ stream->started = 1; -+ break; -+ case SNDRV_PCM_TRIGGER_STOP: -+ aaudio_cmd_stop_io(sdev->a, sdev->dev_id); -+ stream->started = 0; -+ break; -+ default: -+ return -EINVAL; -+ } -+ return 0; -+} -+ -+static snd_pcm_uframes_t aaudio_pcm_pointer(struct snd_pcm_substream *substream) -+{ -+ struct aaudio_stream *stream = aaudio_pcm_stream(substream); -+ ktime_t time_from_start; -+ snd_pcm_sframes_t frames; -+ snd_pcm_sframes_t buffer_time_length; -+ -+ if (!stream->started || stream->waiting_for_first_ts) { -+ pr_warn("aaudio_pcm_pointer while not started\n"); -+ return 0; -+ } -+ -+ /* Approximate the pointer based on the last received timestamp */ -+ time_from_start = ktime_get_boottime() - stream->remote_timestamp; -+ buffer_time_length = NSEC_PER_SEC * substream->runtime->buffer_size / substream->runtime->rate; -+ frames = (ktime_to_ns(time_from_start) % buffer_time_length) * substream->runtime->buffer_size / buffer_time_length; -+ if (ktime_to_ns(time_from_start) < buffer_time_length) { -+ if (frames < stream->frame_min) -+ frames = stream->frame_min; -+ else -+ stream->frame_min = 0; -+ } else { -+ if (ktime_to_ns(time_from_start) < 2 * buffer_time_length) -+ stream->frame_min = frames; -+ else -+ stream->frame_min = 0; /* Heavy desync */ -+ } -+ frames -= stream->latency; -+ if (frames < 0) -+ frames += ((-frames - 1) / substream->runtime->buffer_size + 1) * substream->runtime->buffer_size; -+ return (snd_pcm_uframes_t) frames; -+} -+ -+static struct snd_pcm_ops aaudio_pcm_ops = { -+ .open = aaudio_pcm_open, -+ .close = aaudio_pcm_close, -+ .ioctl = snd_pcm_lib_ioctl, -+ .hw_params = aaudio_pcm_hw_params, -+ .hw_free = aaudio_pcm_hw_free, -+ .prepare = aaudio_pcm_prepare, -+ .trigger = aaudio_pcm_trigger, -+ .pointer = aaudio_pcm_pointer, -+ .mmap = snd_pcm_lib_mmap_iomem -+}; -+ -+int aaudio_create_pcm(struct aaudio_subdevice *sdev) -+{ -+ struct snd_pcm *pcm; -+ struct aaudio_alsa_pcm_id_mapping *id_mapping; -+ int err; -+ -+ if (!sdev->is_pcm || (sdev->in_stream_cnt == 0 && sdev->out_stream_cnt == 0)) { -+ return -EINVAL; -+ } -+ -+ for (id_mapping = aaudio_alsa_id_mappings; id_mapping->name; id_mapping++) { -+ if (!strcmp(sdev->uid, id_mapping->name)) { -+ sdev->alsa_id = id_mapping->alsa_id; -+ break; -+ } -+ } -+ if (!id_mapping->name) -+ sdev->alsa_id = sdev->a->next_alsa_id++; -+ err = snd_pcm_new(sdev->a->card, sdev->uid, sdev->alsa_id, -+ (int) sdev->out_stream_cnt, (int) sdev->in_stream_cnt, &pcm); -+ if (err < 0) -+ return err; -+ pcm->private_data = sdev; -+ pcm->nonatomic = 1; -+ sdev->pcm = pcm; -+ strcpy(pcm->name, sdev->uid); -+ snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_PLAYBACK, &aaudio_pcm_ops); -+ snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_CAPTURE, &aaudio_pcm_ops); -+ return 0; -+} -+ -+static void aaudio_handle_stream_timestamp(struct snd_pcm_substream *substream, ktime_t timestamp) -+{ -+ unsigned long flags; -+ struct aaudio_stream *stream; -+ -+ stream = aaudio_pcm_stream(substream); -+ snd_pcm_stream_lock_irqsave(substream, flags); -+ stream->remote_timestamp = timestamp; -+ if (stream->waiting_for_first_ts) { -+ stream->waiting_for_first_ts = false; -+ snd_pcm_stream_unlock_irqrestore(substream, flags); -+ return; -+ } -+ snd_pcm_stream_unlock_irqrestore(substream, flags); -+ snd_pcm_period_elapsed(substream); -+} -+ -+void aaudio_handle_timestamp(struct aaudio_subdevice *sdev, ktime_t os_timestamp, u64 dev_timestamp) -+{ -+ struct snd_pcm_substream *substream; -+ -+ substream = sdev->pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream; -+ if (substream) -+ aaudio_handle_stream_timestamp(substream, dev_timestamp); -+ substream = sdev->pcm->streams[SNDRV_PCM_STREAM_CAPTURE].substream; -+ if (substream) -+ aaudio_handle_stream_timestamp(substream, os_timestamp); -+} -diff --git a/drivers/staging/apple-bce/audio/pcm.h b/drivers/staging/apple-bce/audio/pcm.h -new file mode 100644 -index 000000000000..ea5f35fbe408 ---- /dev/null -+++ b/drivers/staging/apple-bce/audio/pcm.h -@@ -0,0 +1,16 @@ -+#ifndef AAUDIO_PCM_H -+#define AAUDIO_PCM_H -+ -+#include -+#include -+ -+struct aaudio_subdevice; -+struct aaudio_apple_description; -+struct snd_pcm_hardware; -+ -+int aaudio_create_hw_info(struct aaudio_apple_description *desc, struct snd_pcm_hardware *alsa_hw, size_t buf_size); -+int aaudio_create_pcm(struct aaudio_subdevice *sdev); -+ -+void aaudio_handle_timestamp(struct aaudio_subdevice *sdev, ktime_t os_timestamp, u64 dev_timestamp); -+ -+#endif //AAUDIO_PCM_H -diff --git a/drivers/staging/apple-bce/audio/protocol.c b/drivers/staging/apple-bce/audio/protocol.c -new file mode 100644 -index 000000000000..2314813aeead ---- /dev/null -+++ b/drivers/staging/apple-bce/audio/protocol.c -@@ -0,0 +1,347 @@ -+#include "protocol.h" -+#include "protocol_bce.h" -+#include "audio.h" -+ -+int aaudio_msg_read_base(struct aaudio_msg *msg, struct aaudio_msg_base *base) -+{ -+ if (msg->size < sizeof(struct aaudio_msg_header) + sizeof(struct aaudio_msg_base) * 2) -+ return -EINVAL; -+ *base = *((struct aaudio_msg_base *) ((struct aaudio_msg_header *) msg->data + 1)); -+ return 0; -+} -+ -+#define READ_START(type) \ -+ size_t offset = sizeof(struct aaudio_msg_header) + sizeof(struct aaudio_msg_base); (void)offset; \ -+ if (((struct aaudio_msg_base *) ((struct aaudio_msg_header *) msg->data + 1))->msg != type) \ -+ return -EINVAL; -+#define READ_DEVID_VAR(devid) *devid = ((struct aaudio_msg_header *) msg->data)->device_id -+#define READ_VAL(type) ({ offset += sizeof(type); *((type *) ((u8 *) msg->data + offset - sizeof(type))); }) -+#define READ_VAR(type, var) *var = READ_VAL(type) -+ -+int aaudio_msg_read_start_io_response(struct aaudio_msg *msg) -+{ -+ READ_START(AAUDIO_MSG_START_IO_RESPONSE); -+ return 0; -+} -+ -+int aaudio_msg_read_stop_io_response(struct aaudio_msg *msg) -+{ -+ READ_START(AAUDIO_MSG_STOP_IO_RESPONSE); -+ return 0; -+} -+ -+int aaudio_msg_read_update_timestamp(struct aaudio_msg *msg, aaudio_device_id_t *devid, -+ u64 *timestamp, u64 *update_seed) -+{ -+ READ_START(AAUDIO_MSG_UPDATE_TIMESTAMP); -+ READ_DEVID_VAR(devid); -+ READ_VAR(u64, timestamp); -+ READ_VAR(u64, update_seed); -+ return 0; -+} -+ -+int aaudio_msg_read_get_property_response(struct aaudio_msg *msg, aaudio_object_id_t *obj, -+ struct aaudio_prop_addr *prop, void **data, u64 *data_size) -+{ -+ READ_START(AAUDIO_MSG_GET_PROPERTY_RESPONSE); -+ READ_VAR(aaudio_object_id_t, obj); -+ READ_VAR(u32, &prop->element); -+ READ_VAR(u32, &prop->scope); -+ READ_VAR(u32, &prop->selector); -+ READ_VAR(u64, data_size); -+ *data = ((u8 *) msg->data + offset); -+ /* offset += data_size; */ -+ return 0; -+} -+ -+int aaudio_msg_read_set_property_response(struct aaudio_msg *msg, aaudio_object_id_t *obj) -+{ -+ READ_START(AAUDIO_MSG_SET_PROPERTY_RESPONSE); -+ READ_VAR(aaudio_object_id_t, obj); -+ return 0; -+} -+ -+int aaudio_msg_read_property_listener_response(struct aaudio_msg *msg, aaudio_object_id_t *obj, -+ struct aaudio_prop_addr *prop) -+{ -+ READ_START(AAUDIO_MSG_PROPERTY_LISTENER_RESPONSE); -+ READ_VAR(aaudio_object_id_t, obj); -+ READ_VAR(u32, &prop->element); -+ READ_VAR(u32, &prop->scope); -+ READ_VAR(u32, &prop->selector); -+ return 0; -+} -+ -+int aaudio_msg_read_property_changed(struct aaudio_msg *msg, aaudio_device_id_t *devid, aaudio_object_id_t *obj, -+ struct aaudio_prop_addr *prop) -+{ -+ READ_START(AAUDIO_MSG_PROPERTY_CHANGED); -+ READ_DEVID_VAR(devid); -+ READ_VAR(aaudio_object_id_t, obj); -+ READ_VAR(u32, &prop->element); -+ READ_VAR(u32, &prop->scope); -+ READ_VAR(u32, &prop->selector); -+ return 0; -+} -+ -+int aaudio_msg_read_set_input_stream_address_ranges_response(struct aaudio_msg *msg) -+{ -+ READ_START(AAUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES_RESPONSE); -+ return 0; -+} -+ -+int aaudio_msg_read_get_input_stream_list_response(struct aaudio_msg *msg, aaudio_object_id_t **str_l, u64 *str_cnt) -+{ -+ READ_START(AAUDIO_MSG_GET_INPUT_STREAM_LIST_RESPONSE); -+ READ_VAR(u64, str_cnt); -+ *str_l = (aaudio_device_id_t *) ((u8 *) msg->data + offset); -+ /* offset += str_cnt * sizeof(aaudio_object_id_t); */ -+ return 0; -+} -+ -+int aaudio_msg_read_get_output_stream_list_response(struct aaudio_msg *msg, aaudio_object_id_t **str_l, u64 *str_cnt) -+{ -+ READ_START(AAUDIO_MSG_GET_OUTPUT_STREAM_LIST_RESPONSE); -+ READ_VAR(u64, str_cnt); -+ *str_l = (aaudio_device_id_t *) ((u8 *) msg->data + offset); -+ /* offset += str_cnt * sizeof(aaudio_object_id_t); */ -+ return 0; -+} -+ -+int aaudio_msg_read_set_remote_access_response(struct aaudio_msg *msg) -+{ -+ READ_START(AAUDIO_MSG_SET_REMOTE_ACCESS_RESPONSE); -+ return 0; -+} -+ -+int aaudio_msg_read_get_device_list_response(struct aaudio_msg *msg, aaudio_device_id_t **dev_l, u64 *dev_cnt) -+{ -+ READ_START(AAUDIO_MSG_GET_DEVICE_LIST_RESPONSE); -+ READ_VAR(u64, dev_cnt); -+ *dev_l = (aaudio_device_id_t *) ((u8 *) msg->data + offset); -+ /* offset += dev_cnt * sizeof(aaudio_device_id_t); */ -+ return 0; -+} -+ -+#define WRITE_START_OF_TYPE(typev, devid) \ -+ size_t offset = sizeof(struct aaudio_msg_header); (void) offset; \ -+ ((struct aaudio_msg_header *) msg->data)->type = (typev); \ -+ ((struct aaudio_msg_header *) msg->data)->device_id = (devid); -+#define WRITE_START_COMMAND(devid) WRITE_START_OF_TYPE(AAUDIO_MSG_TYPE_COMMAND, devid) -+#define WRITE_START_RESPONSE() WRITE_START_OF_TYPE(AAUDIO_MSG_TYPE_RESPONSE, 0) -+#define WRITE_START_NOTIFICATION() WRITE_START_OF_TYPE(AAUDIO_MSG_TYPE_NOTIFICATION, 0) -+#define WRITE_VAL(type, value) { *((type *) ((u8 *) msg->data + offset)) = value; offset += sizeof(value); } -+#define WRITE_BIN(value, size) { memcpy((u8 *) msg->data + offset, value, size); offset += size; } -+#define WRITE_BASE(type) WRITE_VAL(u32, type) WRITE_VAL(u32, 0) -+#define WRITE_END() { msg->size = offset; } -+ -+void aaudio_msg_write_start_io(struct aaudio_msg *msg, aaudio_device_id_t dev) -+{ -+ WRITE_START_COMMAND(dev); -+ WRITE_BASE(AAUDIO_MSG_START_IO); -+ WRITE_END(); -+} -+ -+void aaudio_msg_write_stop_io(struct aaudio_msg *msg, aaudio_device_id_t dev) -+{ -+ WRITE_START_COMMAND(dev); -+ WRITE_BASE(AAUDIO_MSG_STOP_IO); -+ WRITE_END(); -+} -+ -+void aaudio_msg_write_get_property(struct aaudio_msg *msg, aaudio_device_id_t dev, aaudio_object_id_t obj, -+ struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size) -+{ -+ WRITE_START_COMMAND(dev); -+ WRITE_BASE(AAUDIO_MSG_GET_PROPERTY); -+ WRITE_VAL(aaudio_object_id_t, obj); -+ WRITE_VAL(u32, prop.element); -+ WRITE_VAL(u32, prop.scope); -+ WRITE_VAL(u32, prop.selector); -+ WRITE_VAL(u64, qualifier_size); -+ WRITE_BIN(qualifier, qualifier_size); -+ WRITE_END(); -+} -+ -+void aaudio_msg_write_set_property(struct aaudio_msg *msg, aaudio_device_id_t dev, aaudio_object_id_t obj, -+ struct aaudio_prop_addr prop, void *data, u64 data_size, void *qualifier, u64 qualifier_size) -+{ -+ WRITE_START_COMMAND(dev); -+ WRITE_BASE(AAUDIO_MSG_SET_PROPERTY); -+ WRITE_VAL(aaudio_object_id_t, obj); -+ WRITE_VAL(u32, prop.element); -+ WRITE_VAL(u32, prop.scope); -+ WRITE_VAL(u32, prop.selector); -+ WRITE_VAL(u64, data_size); -+ WRITE_BIN(data, data_size); -+ WRITE_VAL(u64, qualifier_size); -+ WRITE_BIN(qualifier, qualifier_size); -+ WRITE_END(); -+} -+ -+void aaudio_msg_write_property_listener(struct aaudio_msg *msg, aaudio_device_id_t dev, aaudio_object_id_t obj, -+ struct aaudio_prop_addr prop) -+{ -+ WRITE_START_COMMAND(dev); -+ WRITE_BASE(AAUDIO_MSG_PROPERTY_LISTENER); -+ WRITE_VAL(aaudio_object_id_t, obj); -+ WRITE_VAL(u32, prop.element); -+ WRITE_VAL(u32, prop.scope); -+ WRITE_VAL(u32, prop.selector); -+ WRITE_END(); -+} -+ -+void aaudio_msg_write_set_input_stream_address_ranges(struct aaudio_msg *msg, aaudio_device_id_t devid) -+{ -+ WRITE_START_COMMAND(devid); -+ WRITE_BASE(AAUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES); -+ WRITE_END(); -+} -+ -+void aaudio_msg_write_get_input_stream_list(struct aaudio_msg *msg, aaudio_device_id_t devid) -+{ -+ WRITE_START_COMMAND(devid); -+ WRITE_BASE(AAUDIO_MSG_GET_INPUT_STREAM_LIST); -+ WRITE_END(); -+} -+ -+void aaudio_msg_write_get_output_stream_list(struct aaudio_msg *msg, aaudio_device_id_t devid) -+{ -+ WRITE_START_COMMAND(devid); -+ WRITE_BASE(AAUDIO_MSG_GET_OUTPUT_STREAM_LIST); -+ WRITE_END(); -+} -+ -+void aaudio_msg_write_set_remote_access(struct aaudio_msg *msg, u64 mode) -+{ -+ WRITE_START_COMMAND(0); -+ WRITE_BASE(AAUDIO_MSG_SET_REMOTE_ACCESS); -+ WRITE_VAL(u64, mode); -+ WRITE_END(); -+} -+ -+void aaudio_msg_write_alive_notification(struct aaudio_msg *msg, u32 proto_ver, u32 msg_ver) -+{ -+ WRITE_START_NOTIFICATION(); -+ WRITE_BASE(AAUDIO_MSG_NOTIFICATION_ALIVE); -+ WRITE_VAL(u32, proto_ver); -+ WRITE_VAL(u32, msg_ver); -+ WRITE_END(); -+} -+ -+void aaudio_msg_write_update_timestamp_response(struct aaudio_msg *msg) -+{ -+ WRITE_START_RESPONSE(); -+ WRITE_BASE(AAUDIO_MSG_UPDATE_TIMESTAMP_RESPONSE); -+ WRITE_END(); -+} -+ -+void aaudio_msg_write_get_device_list(struct aaudio_msg *msg) -+{ -+ WRITE_START_COMMAND(0); -+ WRITE_BASE(AAUDIO_MSG_GET_DEVICE_LIST); -+ WRITE_END(); -+} -+ -+#define CMD_SHARED_VARS_NO_REPLY \ -+ int status = 0; \ -+ struct aaudio_send_ctx sctx; -+#define CMD_SHARED_VARS \ -+ CMD_SHARED_VARS_NO_REPLY \ -+ struct aaudio_msg reply = aaudio_reply_alloc(); \ -+ struct aaudio_msg *buf = &reply; -+#define CMD_SEND_REQUEST(fn, ...) \ -+ if ((status = aaudio_send_cmd_sync(a, &sctx, buf, 500, fn, ##__VA_ARGS__))) \ -+ return status; -+#define CMD_DEF_SHARED_AND_SEND(fn, ...) \ -+ CMD_SHARED_VARS \ -+ CMD_SEND_REQUEST(fn, ##__VA_ARGS__); -+#define CMD_DEF_SHARED_NO_REPLY_AND_SEND(fn, ...) \ -+ CMD_SHARED_VARS_NO_REPLY \ -+ CMD_SEND_REQUEST(fn, ##__VA_ARGS__); -+#define CMD_HNDL_REPLY_NO_FREE(fn, ...) \ -+ status = fn(buf, ##__VA_ARGS__); \ -+ return status; -+#define CMD_HNDL_REPLY_AND_FREE(fn, ...) \ -+ status = fn(buf, ##__VA_ARGS__); \ -+ aaudio_reply_free(&reply); \ -+ return status; -+ -+int aaudio_cmd_start_io(struct aaudio_device *a, aaudio_device_id_t devid) -+{ -+ CMD_DEF_SHARED_AND_SEND(aaudio_msg_write_start_io, devid); -+ CMD_HNDL_REPLY_AND_FREE(aaudio_msg_read_start_io_response); -+} -+int aaudio_cmd_stop_io(struct aaudio_device *a, aaudio_device_id_t devid) -+{ -+ CMD_DEF_SHARED_AND_SEND(aaudio_msg_write_stop_io, devid); -+ CMD_HNDL_REPLY_AND_FREE(aaudio_msg_read_stop_io_response); -+} -+int aaudio_cmd_get_property(struct aaudio_device *a, struct aaudio_msg *buf, -+ aaudio_device_id_t devid, aaudio_object_id_t obj, -+ struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size, void **data, u64 *data_size) -+{ -+ CMD_DEF_SHARED_NO_REPLY_AND_SEND(aaudio_msg_write_get_property, devid, obj, prop, qualifier, qualifier_size); -+ CMD_HNDL_REPLY_NO_FREE(aaudio_msg_read_get_property_response, &obj, &prop, data, data_size); -+} -+int aaudio_cmd_get_primitive_property(struct aaudio_device *a, -+ aaudio_device_id_t devid, aaudio_object_id_t obj, -+ struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size) -+{ -+ int status; -+ struct aaudio_msg reply = aaudio_reply_alloc(); -+ void *r_data; -+ u64 r_data_size; -+ if ((status = aaudio_cmd_get_property(a, &reply, devid, obj, prop, qualifier, qualifier_size, -+ &r_data, &r_data_size))) -+ goto finish; -+ if (r_data_size != data_size) { -+ status = -EINVAL; -+ goto finish; -+ } -+ memcpy(data, r_data, data_size); -+finish: -+ aaudio_reply_free(&reply); -+ return status; -+} -+int aaudio_cmd_set_property(struct aaudio_device *a, aaudio_device_id_t devid, aaudio_object_id_t obj, -+ struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size) -+{ -+ CMD_DEF_SHARED_AND_SEND(aaudio_msg_write_set_property, devid, obj, prop, data, data_size, -+ qualifier, qualifier_size); -+ CMD_HNDL_REPLY_AND_FREE(aaudio_msg_read_set_property_response, &obj); -+} -+int aaudio_cmd_property_listener(struct aaudio_device *a, aaudio_device_id_t devid, aaudio_object_id_t obj, -+ struct aaudio_prop_addr prop) -+{ -+ CMD_DEF_SHARED_AND_SEND(aaudio_msg_write_property_listener, devid, obj, prop); -+ CMD_HNDL_REPLY_AND_FREE(aaudio_msg_read_property_listener_response, &obj, &prop); -+} -+int aaudio_cmd_set_input_stream_address_ranges(struct aaudio_device *a, aaudio_device_id_t devid) -+{ -+ CMD_DEF_SHARED_AND_SEND(aaudio_msg_write_set_input_stream_address_ranges, devid); -+ CMD_HNDL_REPLY_AND_FREE(aaudio_msg_read_set_input_stream_address_ranges_response); -+} -+int aaudio_cmd_get_input_stream_list(struct aaudio_device *a, struct aaudio_msg *buf, aaudio_device_id_t devid, -+ aaudio_object_id_t **str_l, u64 *str_cnt) -+{ -+ CMD_DEF_SHARED_NO_REPLY_AND_SEND(aaudio_msg_write_get_input_stream_list, devid); -+ CMD_HNDL_REPLY_NO_FREE(aaudio_msg_read_get_input_stream_list_response, str_l, str_cnt); -+} -+int aaudio_cmd_get_output_stream_list(struct aaudio_device *a, struct aaudio_msg *buf, aaudio_device_id_t devid, -+ aaudio_object_id_t **str_l, u64 *str_cnt) -+{ -+ CMD_DEF_SHARED_NO_REPLY_AND_SEND(aaudio_msg_write_get_output_stream_list, devid); -+ CMD_HNDL_REPLY_NO_FREE(aaudio_msg_read_get_output_stream_list_response, str_l, str_cnt); -+} -+int aaudio_cmd_set_remote_access(struct aaudio_device *a, u64 mode) -+{ -+ CMD_DEF_SHARED_AND_SEND(aaudio_msg_write_set_remote_access, mode); -+ CMD_HNDL_REPLY_AND_FREE(aaudio_msg_read_set_remote_access_response); -+} -+int aaudio_cmd_get_device_list(struct aaudio_device *a, struct aaudio_msg *buf, -+ aaudio_device_id_t **dev_l, u64 *dev_cnt) -+{ -+ CMD_DEF_SHARED_NO_REPLY_AND_SEND(aaudio_msg_write_get_device_list); -+ CMD_HNDL_REPLY_NO_FREE(aaudio_msg_read_get_device_list_response, dev_l, dev_cnt); -+} -\ No newline at end of file -diff --git a/drivers/staging/apple-bce/audio/protocol.h b/drivers/staging/apple-bce/audio/protocol.h -new file mode 100644 -index 000000000000..3427486f3f57 ---- /dev/null -+++ b/drivers/staging/apple-bce/audio/protocol.h -@@ -0,0 +1,147 @@ -+#ifndef AAUDIO_PROTOCOL_H -+#define AAUDIO_PROTOCOL_H -+ -+#include -+ -+struct aaudio_device; -+ -+typedef u64 aaudio_device_id_t; -+typedef u64 aaudio_object_id_t; -+ -+struct aaudio_msg { -+ void *data; -+ size_t size; -+}; -+ -+struct __attribute__((packed)) aaudio_msg_header { -+ char tag[4]; -+ u8 type; -+ aaudio_device_id_t device_id; // Idk, use zero for commands? -+}; -+struct __attribute__((packed)) aaudio_msg_base { -+ u32 msg; -+ u32 status; -+}; -+ -+struct aaudio_prop_addr { -+ u32 scope; -+ u32 selector; -+ u32 element; -+}; -+#define AAUDIO_PROP(scope, sel, el) (struct aaudio_prop_addr) { scope, sel, el } -+ -+enum { -+ AAUDIO_MSG_TYPE_COMMAND = 1, -+ AAUDIO_MSG_TYPE_RESPONSE = 2, -+ AAUDIO_MSG_TYPE_NOTIFICATION = 3 -+}; -+ -+enum { -+ AAUDIO_MSG_START_IO = 0, -+ AAUDIO_MSG_START_IO_RESPONSE = 1, -+ AAUDIO_MSG_STOP_IO = 2, -+ AAUDIO_MSG_STOP_IO_RESPONSE = 3, -+ AAUDIO_MSG_UPDATE_TIMESTAMP = 4, -+ AAUDIO_MSG_GET_PROPERTY = 7, -+ AAUDIO_MSG_GET_PROPERTY_RESPONSE = 8, -+ AAUDIO_MSG_SET_PROPERTY = 9, -+ AAUDIO_MSG_SET_PROPERTY_RESPONSE = 10, -+ AAUDIO_MSG_PROPERTY_LISTENER = 11, -+ AAUDIO_MSG_PROPERTY_LISTENER_RESPONSE = 12, -+ AAUDIO_MSG_PROPERTY_CHANGED = 13, -+ AAUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES = 18, -+ AAUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES_RESPONSE = 19, -+ AAUDIO_MSG_GET_INPUT_STREAM_LIST = 24, -+ AAUDIO_MSG_GET_INPUT_STREAM_LIST_RESPONSE = 25, -+ AAUDIO_MSG_GET_OUTPUT_STREAM_LIST = 26, -+ AAUDIO_MSG_GET_OUTPUT_STREAM_LIST_RESPONSE = 27, -+ AAUDIO_MSG_SET_REMOTE_ACCESS = 32, -+ AAUDIO_MSG_SET_REMOTE_ACCESS_RESPONSE = 33, -+ AAUDIO_MSG_UPDATE_TIMESTAMP_RESPONSE = 34, -+ -+ AAUDIO_MSG_NOTIFICATION_ALIVE = 100, -+ AAUDIO_MSG_GET_DEVICE_LIST = 101, -+ AAUDIO_MSG_GET_DEVICE_LIST_RESPONSE = 102, -+ AAUDIO_MSG_NOTIFICATION_BOOT = 104 -+}; -+ -+enum { -+ AAUDIO_REMOTE_ACCESS_OFF = 0, -+ AAUDIO_REMOTE_ACCESS_ON = 2 -+}; -+ -+enum { -+ AAUDIO_PROP_SCOPE_GLOBAL = 0x676c6f62, // 'glob' -+ AAUDIO_PROP_SCOPE_INPUT = 0x696e7074, // 'inpt' -+ AAUDIO_PROP_SCOPE_OUTPUT = 0x6f757470 // 'outp' -+}; -+ -+enum { -+ AAUDIO_PROP_UID = 0x75696420, // 'uid ' -+ AAUDIO_PROP_BOOL_VALUE = 0x6263766c, // 'bcvl' -+ AAUDIO_PROP_JACK_PLUGGED = 0x6a61636b, // 'jack' -+ AAUDIO_PROP_SEL_VOLUME = 0x64656176, // 'deav' -+ AAUDIO_PROP_LATENCY = 0x6c746e63, // 'ltnc' -+ AAUDIO_PROP_PHYS_FORMAT = 0x70667420 // 'pft ' -+}; -+ -+int aaudio_msg_read_base(struct aaudio_msg *msg, struct aaudio_msg_base *base); -+ -+int aaudio_msg_read_start_io_response(struct aaudio_msg *msg); -+int aaudio_msg_read_stop_io_response(struct aaudio_msg *msg); -+int aaudio_msg_read_update_timestamp(struct aaudio_msg *msg, aaudio_device_id_t *devid, -+ u64 *timestamp, u64 *update_seed); -+int aaudio_msg_read_get_property_response(struct aaudio_msg *msg, aaudio_object_id_t *obj, -+ struct aaudio_prop_addr *prop, void **data, u64 *data_size); -+int aaudio_msg_read_set_property_response(struct aaudio_msg *msg, aaudio_object_id_t *obj); -+int aaudio_msg_read_property_listener_response(struct aaudio_msg *msg,aaudio_object_id_t *obj, -+ struct aaudio_prop_addr *prop); -+int aaudio_msg_read_property_changed(struct aaudio_msg *msg, aaudio_device_id_t *devid, aaudio_object_id_t *obj, -+ struct aaudio_prop_addr *prop); -+int aaudio_msg_read_set_input_stream_address_ranges_response(struct aaudio_msg *msg); -+int aaudio_msg_read_get_input_stream_list_response(struct aaudio_msg *msg, aaudio_object_id_t **str_l, u64 *str_cnt); -+int aaudio_msg_read_get_output_stream_list_response(struct aaudio_msg *msg, aaudio_object_id_t **str_l, u64 *str_cnt); -+int aaudio_msg_read_set_remote_access_response(struct aaudio_msg *msg); -+int aaudio_msg_read_get_device_list_response(struct aaudio_msg *msg, aaudio_device_id_t **dev_l, u64 *dev_cnt); -+ -+void aaudio_msg_write_start_io(struct aaudio_msg *msg, aaudio_device_id_t dev); -+void aaudio_msg_write_stop_io(struct aaudio_msg *msg, aaudio_device_id_t dev); -+void aaudio_msg_write_get_property(struct aaudio_msg *msg, aaudio_device_id_t dev, aaudio_object_id_t obj, -+ struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size); -+void aaudio_msg_write_set_property(struct aaudio_msg *msg, aaudio_device_id_t dev, aaudio_object_id_t obj, -+ struct aaudio_prop_addr prop, void *data, u64 data_size, void *qualifier, u64 qualifier_size); -+void aaudio_msg_write_property_listener(struct aaudio_msg *msg, aaudio_device_id_t dev, aaudio_object_id_t obj, -+ struct aaudio_prop_addr prop); -+void aaudio_msg_write_set_input_stream_address_ranges(struct aaudio_msg *msg, aaudio_device_id_t devid); -+void aaudio_msg_write_get_input_stream_list(struct aaudio_msg *msg, aaudio_device_id_t devid); -+void aaudio_msg_write_get_output_stream_list(struct aaudio_msg *msg, aaudio_device_id_t devid); -+void aaudio_msg_write_set_remote_access(struct aaudio_msg *msg, u64 mode); -+void aaudio_msg_write_alive_notification(struct aaudio_msg *msg, u32 proto_ver, u32 msg_ver); -+void aaudio_msg_write_update_timestamp_response(struct aaudio_msg *msg); -+void aaudio_msg_write_get_device_list(struct aaudio_msg *msg); -+ -+ -+int aaudio_cmd_start_io(struct aaudio_device *a, aaudio_device_id_t devid); -+int aaudio_cmd_stop_io(struct aaudio_device *a, aaudio_device_id_t devid); -+int aaudio_cmd_get_property(struct aaudio_device *a, struct aaudio_msg *buf, -+ aaudio_device_id_t devid, aaudio_object_id_t obj, -+ struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size, void **data, u64 *data_size); -+int aaudio_cmd_get_primitive_property(struct aaudio_device *a, -+ aaudio_device_id_t devid, aaudio_object_id_t obj, -+ struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size); -+int aaudio_cmd_set_property(struct aaudio_device *a, aaudio_device_id_t devid, aaudio_object_id_t obj, -+ struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size); -+int aaudio_cmd_property_listener(struct aaudio_device *a, aaudio_device_id_t devid, aaudio_object_id_t obj, -+ struct aaudio_prop_addr prop); -+int aaudio_cmd_set_input_stream_address_ranges(struct aaudio_device *a, aaudio_device_id_t devid); -+int aaudio_cmd_get_input_stream_list(struct aaudio_device *a, struct aaudio_msg *buf, aaudio_device_id_t devid, -+ aaudio_object_id_t **str_l, u64 *str_cnt); -+int aaudio_cmd_get_output_stream_list(struct aaudio_device *a, struct aaudio_msg *buf, aaudio_device_id_t devid, -+ aaudio_object_id_t **str_l, u64 *str_cnt); -+int aaudio_cmd_set_remote_access(struct aaudio_device *a, u64 mode); -+int aaudio_cmd_get_device_list(struct aaudio_device *a, struct aaudio_msg *buf, -+ aaudio_device_id_t **dev_l, u64 *dev_cnt); -+ -+ -+ -+#endif //AAUDIO_PROTOCOL_H -diff --git a/drivers/staging/apple-bce/audio/protocol_bce.c b/drivers/staging/apple-bce/audio/protocol_bce.c -new file mode 100644 -index 000000000000..28f2dfd44d67 ---- /dev/null -+++ b/drivers/staging/apple-bce/audio/protocol_bce.c -@@ -0,0 +1,226 @@ -+#include "protocol_bce.h" -+ -+#include "audio.h" -+ -+static void aaudio_bce_out_queue_completion(struct bce_queue_sq *sq); -+static void aaudio_bce_in_queue_completion(struct bce_queue_sq *sq); -+static int aaudio_bce_queue_init(struct aaudio_device *dev, struct aaudio_bce_queue *q, const char *name, int direction, -+ bce_sq_completion cfn); -+void aaudio_bce_in_queue_submit_pending(struct aaudio_bce_queue *q, size_t count); -+ -+int aaudio_bce_init(struct aaudio_device *dev) -+{ -+ int status; -+ struct aaudio_bce *bce = &dev->bcem; -+ bce->cq = bce_create_cq(dev->bce, 0x80); -+ spin_lock_init(&bce->spinlock); -+ if (!bce->cq) -+ return -EINVAL; -+ if ((status = aaudio_bce_queue_init(dev, &bce->qout, "com.apple.BridgeAudio.IntelToARM", DMA_TO_DEVICE, -+ aaudio_bce_out_queue_completion))) { -+ return status; -+ } -+ if ((status = aaudio_bce_queue_init(dev, &bce->qin, "com.apple.BridgeAudio.ARMToIntel", DMA_FROM_DEVICE, -+ aaudio_bce_in_queue_completion))) { -+ return status; -+ } -+ aaudio_bce_in_queue_submit_pending(&bce->qin, bce->qin.el_count); -+ return 0; -+} -+ -+int aaudio_bce_queue_init(struct aaudio_device *dev, struct aaudio_bce_queue *q, const char *name, int direction, -+ bce_sq_completion cfn) -+{ -+ q->cq = dev->bcem.cq; -+ q->el_size = AAUDIO_BCE_QUEUE_ELEMENT_SIZE; -+ q->el_count = AAUDIO_BCE_QUEUE_ELEMENT_COUNT; -+ /* NOTE: The Apple impl uses 0x80 as the queue size, however we use 21 (in fact 20) to simplify the impl */ -+ q->sq = bce_create_sq(dev->bce, q->cq, name, (u32) (q->el_count + 1), direction, cfn, dev); -+ if (!q->sq) -+ return -EINVAL; -+ -+ q->data = dma_alloc_coherent(&dev->bce->pci->dev, q->el_size * q->el_count, &q->dma_addr, GFP_KERNEL); -+ if (!q->data) { -+ bce_destroy_sq(dev->bce, q->sq); -+ return -EINVAL; -+ } -+ return 0; -+} -+ -+static void aaudio_send_create_tag(struct aaudio_bce *b, int *tagn, char tag[4]) -+{ -+ char tag_zero[5]; -+ b->tag_num = (b->tag_num + 1) % AAUDIO_BCE_QUEUE_TAG_COUNT; -+ *tagn = b->tag_num; -+ snprintf(tag_zero, 5, "S%03d", b->tag_num); -+ *((u32 *) tag) = *((u32 *) tag_zero); -+} -+ -+int __aaudio_send_prepare(struct aaudio_bce *b, struct aaudio_send_ctx *ctx, char *tag) -+{ -+ int status; -+ size_t index; -+ void *dptr; -+ struct aaudio_msg_header *header; -+ if ((status = bce_reserve_submission(b->qout.sq, &ctx->timeout))) -+ return status; -+ spin_lock_irqsave(&b->spinlock, ctx->irq_flags); -+ index = b->qout.data_tail; -+ dptr = (u8 *) b->qout.data + index * b->qout.el_size; -+ ctx->msg.data = dptr; -+ header = dptr; -+ if (tag) -+ *((u32 *) header->tag) = *((u32 *) tag); -+ else -+ aaudio_send_create_tag(b, &ctx->tag_n, header->tag); -+ return 0; -+} -+ -+void __aaudio_send(struct aaudio_bce *b, struct aaudio_send_ctx *ctx) -+{ -+ struct bce_qe_submission *s = bce_next_submission(b->qout.sq); -+#ifdef DEBUG -+ pr_debug("aaudio: Sending command data\n"); -+ print_hex_dump(KERN_DEBUG, "aaudio:OUT ", DUMP_PREFIX_NONE, 32, 1, ctx->msg.data, ctx->msg.size, true); -+#endif -+ bce_set_submission_single(s, b->qout.dma_addr + (dma_addr_t) (ctx->msg.data - b->qout.data), ctx->msg.size); -+ bce_submit_to_device(b->qout.sq); -+ b->qout.data_tail = (b->qout.data_tail + 1) % b->qout.el_count; -+ spin_unlock_irqrestore(&b->spinlock, ctx->irq_flags); -+} -+ -+int __aaudio_send_cmd_sync(struct aaudio_bce *b, struct aaudio_send_ctx *ctx, struct aaudio_msg *reply) -+{ -+ struct aaudio_bce_queue_entry ent; -+ DECLARE_COMPLETION_ONSTACK(cmpl); -+ ent.msg = reply; -+ ent.cmpl = &cmpl; -+ b->pending_entries[ctx->tag_n] = &ent; -+ __aaudio_send(b, ctx); /* unlocks the spinlock */ -+ ctx->timeout = wait_for_completion_timeout(&cmpl, ctx->timeout); -+ if (ctx->timeout == 0) { -+ /* Remove the pending queue entry; this will be normally handled by the completion route but -+ * during a timeout it won't */ -+ spin_lock_irqsave(&b->spinlock, ctx->irq_flags); -+ if (b->pending_entries[ctx->tag_n] == &ent) -+ b->pending_entries[ctx->tag_n] = NULL; -+ spin_unlock_irqrestore(&b->spinlock, ctx->irq_flags); -+ return -ETIMEDOUT; -+ } -+ return 0; -+} -+ -+static void aaudio_handle_reply(struct aaudio_bce *b, struct aaudio_msg *reply) -+{ -+ const char *tag; -+ int tagn; -+ unsigned long irq_flags; -+ char tag_zero[5]; -+ struct aaudio_bce_queue_entry *entry; -+ -+ tag = ((struct aaudio_msg_header *) reply->data)->tag; -+ if (tag[0] != 'S') { -+ pr_err("aaudio_handle_reply: Unexpected tag: %.4s\n", tag); -+ return; -+ } -+ *((u32 *) tag_zero) = *((u32 *) tag); -+ tag_zero[4] = 0; -+ if (kstrtoint(&tag_zero[1], 10, &tagn)) { -+ pr_err("aaudio_handle_reply: Tag parse failed: %.4s\n", tag); -+ return; -+ } -+ -+ spin_lock_irqsave(&b->spinlock, irq_flags); -+ entry = b->pending_entries[tagn]; -+ if (entry) { -+ if (reply->size < entry->msg->size) -+ entry->msg->size = reply->size; -+ memcpy(entry->msg->data, reply->data, entry->msg->size); -+ complete(entry->cmpl); -+ -+ b->pending_entries[tagn] = NULL; -+ } else { -+ pr_err("aaudio_handle_reply: No queued item found for tag: %.4s\n", tag); -+ } -+ spin_unlock_irqrestore(&b->spinlock, irq_flags); -+} -+ -+static void aaudio_bce_out_queue_completion(struct bce_queue_sq *sq) -+{ -+ while (bce_next_completion(sq)) { -+ //pr_info("aaudio: Send confirmed\n"); -+ bce_notify_submission_complete(sq); -+ } -+} -+ -+static void aaudio_bce_in_queue_handle_msg(struct aaudio_device *a, struct aaudio_msg *msg); -+ -+static void aaudio_bce_in_queue_completion(struct bce_queue_sq *sq) -+{ -+ struct aaudio_msg msg; -+ struct aaudio_device *dev = sq->userdata; -+ struct aaudio_bce_queue *q = &dev->bcem.qin; -+ struct bce_sq_completion_data *c; -+ size_t cnt = 0; -+ -+ mb(); -+ while ((c = bce_next_completion(sq))) { -+ msg.data = (u8 *) q->data + q->data_head * q->el_size; -+ msg.size = c->data_size; -+#ifdef DEBUG -+ pr_debug("aaudio: Received command data %llx\n", c->data_size); -+ print_hex_dump(KERN_DEBUG, "aaudio:IN ", DUMP_PREFIX_NONE, 32, 1, msg.data, min(msg.size, 128UL), true); -+#endif -+ aaudio_bce_in_queue_handle_msg(dev, &msg); -+ -+ q->data_head = (q->data_head + 1) % q->el_count; -+ -+ bce_notify_submission_complete(sq); -+ ++cnt; -+ } -+ aaudio_bce_in_queue_submit_pending(q, cnt); -+} -+ -+static void aaudio_bce_in_queue_handle_msg(struct aaudio_device *a, struct aaudio_msg *msg) -+{ -+ struct aaudio_msg_header *header = (struct aaudio_msg_header *) msg->data; -+ if (msg->size < sizeof(struct aaudio_msg_header)) { -+ pr_err("aaudio: Msg size smaller than header (%lx)", msg->size); -+ return; -+ } -+ if (header->type == AAUDIO_MSG_TYPE_RESPONSE) { -+ aaudio_handle_reply(&a->bcem, msg); -+ } else if (header->type == AAUDIO_MSG_TYPE_COMMAND) { -+ aaudio_handle_command(a, msg); -+ } else if (header->type == AAUDIO_MSG_TYPE_NOTIFICATION) { -+ aaudio_handle_notification(a, msg); -+ } -+} -+ -+void aaudio_bce_in_queue_submit_pending(struct aaudio_bce_queue *q, size_t count) -+{ -+ struct bce_qe_submission *s; -+ while (count--) { -+ if (bce_reserve_submission(q->sq, NULL)) { -+ pr_err("aaudio: Failed to reserve an event queue submission\n"); -+ break; -+ } -+ s = bce_next_submission(q->sq); -+ bce_set_submission_single(s, q->dma_addr + (dma_addr_t) (q->data_tail * q->el_size), q->el_size); -+ q->data_tail = (q->data_tail + 1) % q->el_count; -+ } -+ bce_submit_to_device(q->sq); -+} -+ -+struct aaudio_msg aaudio_reply_alloc(void) -+{ -+ struct aaudio_msg ret; -+ ret.size = AAUDIO_BCE_QUEUE_ELEMENT_SIZE; -+ ret.data = kmalloc(ret.size, GFP_KERNEL); -+ return ret; -+} -+ -+void aaudio_reply_free(struct aaudio_msg *reply) -+{ -+ kfree(reply->data); -+} -diff --git a/drivers/staging/apple-bce/audio/protocol_bce.h b/drivers/staging/apple-bce/audio/protocol_bce.h -new file mode 100644 -index 000000000000..14d26c05ddf9 ---- /dev/null -+++ b/drivers/staging/apple-bce/audio/protocol_bce.h -@@ -0,0 +1,72 @@ -+#ifndef AAUDIO_PROTOCOL_BCE_H -+#define AAUDIO_PROTOCOL_BCE_H -+ -+#include "protocol.h" -+#include "../queue.h" -+ -+#define AAUDIO_BCE_QUEUE_ELEMENT_SIZE 0x1000 -+#define AAUDIO_BCE_QUEUE_ELEMENT_COUNT 20 -+ -+#define AAUDIO_BCE_QUEUE_TAG_COUNT 1000 -+ -+struct aaudio_device; -+ -+struct aaudio_bce_queue_entry { -+ struct aaudio_msg *msg; -+ struct completion *cmpl; -+}; -+struct aaudio_bce_queue { -+ struct bce_queue_cq *cq; -+ struct bce_queue_sq *sq; -+ void *data; -+ dma_addr_t dma_addr; -+ size_t data_head, data_tail; -+ size_t el_size, el_count; -+}; -+struct aaudio_bce { -+ struct bce_queue_cq *cq; -+ struct aaudio_bce_queue qin; -+ struct aaudio_bce_queue qout; -+ int tag_num; -+ struct aaudio_bce_queue_entry *pending_entries[AAUDIO_BCE_QUEUE_TAG_COUNT]; -+ struct spinlock spinlock; -+}; -+ -+struct aaudio_send_ctx { -+ int status; -+ int tag_n; -+ unsigned long irq_flags; -+ struct aaudio_msg msg; -+ unsigned long timeout; -+}; -+ -+int aaudio_bce_init(struct aaudio_device *dev); -+int __aaudio_send_prepare(struct aaudio_bce *b, struct aaudio_send_ctx *ctx, char *tag); -+void __aaudio_send(struct aaudio_bce *b, struct aaudio_send_ctx *ctx); -+int __aaudio_send_cmd_sync(struct aaudio_bce *b, struct aaudio_send_ctx *ctx, struct aaudio_msg *reply); -+ -+#define aaudio_send_with_tag(a, ctx, tag, tout, fn, ...) ({ \ -+ (ctx)->timeout = msecs_to_jiffies(tout); \ -+ (ctx)->status = __aaudio_send_prepare(&(a)->bcem, (ctx), (tag)); \ -+ if (!(ctx)->status) { \ -+ fn(&(ctx)->msg, ##__VA_ARGS__); \ -+ __aaudio_send(&(a)->bcem, (ctx)); \ -+ } \ -+ (ctx)->status; \ -+}) -+#define aaudio_send(a, ctx, tout, fn, ...) aaudio_send_with_tag(a, ctx, NULL, tout, fn, ##__VA_ARGS__) -+ -+#define aaudio_send_cmd_sync(a, ctx, reply, tout, fn, ...) ({ \ -+ (ctx)->timeout = msecs_to_jiffies(tout); \ -+ (ctx)->status = __aaudio_send_prepare(&(a)->bcem, (ctx), NULL); \ -+ if (!(ctx)->status) { \ -+ fn(&(ctx)->msg, ##__VA_ARGS__); \ -+ (ctx)->status = __aaudio_send_cmd_sync(&(a)->bcem, (ctx), (reply)); \ -+ } \ -+ (ctx)->status; \ -+}) -+ -+struct aaudio_msg aaudio_reply_alloc(void); -+void aaudio_reply_free(struct aaudio_msg *reply); -+ -+#endif //AAUDIO_PROTOCOL_BCE_H -diff --git a/drivers/staging/apple-bce/mailbox.c b/drivers/staging/apple-bce/mailbox.c -new file mode 100644 -index 000000000000..e24bd35215c0 ---- /dev/null -+++ b/drivers/staging/apple-bce/mailbox.c -@@ -0,0 +1,151 @@ -+#include "mailbox.h" -+#include -+#include "apple_bce.h" -+ -+#define REG_MBOX_OUT_BASE 0x820 -+#define REG_MBOX_REPLY_COUNTER 0x108 -+#define REG_MBOX_REPLY_BASE 0x810 -+#define REG_TIMESTAMP_BASE 0xC000 -+ -+#define BCE_MBOX_TIMEOUT_MS 200 -+ -+void bce_mailbox_init(struct bce_mailbox *mb, void __iomem *reg_mb) -+{ -+ mb->reg_mb = reg_mb; -+ init_completion(&mb->mb_completion); -+} -+ -+int bce_mailbox_send(struct bce_mailbox *mb, u64 msg, u64* recv) -+{ -+ u32 __iomem *regb; -+ -+ if (atomic_cmpxchg(&mb->mb_status, 0, 1) != 0) { -+ return -EEXIST; // We don't support two messages at once -+ } -+ reinit_completion(&mb->mb_completion); -+ -+ pr_debug("bce_mailbox_send: %llx\n", msg); -+ regb = (u32*) ((u8*) mb->reg_mb + REG_MBOX_OUT_BASE); -+ iowrite32((u32) msg, regb); -+ iowrite32((u32) (msg >> 32), regb + 1); -+ iowrite32(0, regb + 2); -+ iowrite32(0, regb + 3); -+ -+ wait_for_completion_timeout(&mb->mb_completion, msecs_to_jiffies(BCE_MBOX_TIMEOUT_MS)); -+ if (atomic_read(&mb->mb_status) != 2) { // Didn't get the reply -+ atomic_set(&mb->mb_status, 0); -+ return -ETIMEDOUT; -+ } -+ -+ *recv = mb->mb_result; -+ pr_debug("bce_mailbox_send: reply %llx\n", *recv); -+ -+ atomic_set(&mb->mb_status, 0); -+ return 0; -+} -+ -+static int bce_mailbox_retrive_response(struct bce_mailbox *mb) -+{ -+ u32 __iomem *regb; -+ u32 lo, hi; -+ int count, counter; -+ u32 res = ioread32((u8*) mb->reg_mb + REG_MBOX_REPLY_COUNTER); -+ count = (res >> 20) & 0xf; -+ counter = count; -+ pr_debug("bce_mailbox_retrive_response count=%i\n", count); -+ while (counter--) { -+ regb = (u32*) ((u8*) mb->reg_mb + REG_MBOX_REPLY_BASE); -+ lo = ioread32(regb); -+ hi = ioread32(regb + 1); -+ ioread32(regb + 2); -+ ioread32(regb + 3); -+ pr_debug("bce_mailbox_retrive_response %llx\n", ((u64) hi << 32) | lo); -+ mb->mb_result = ((u64) hi << 32) | lo; -+ } -+ return count > 0 ? 0 : -ENODATA; -+} -+ -+int bce_mailbox_handle_interrupt(struct bce_mailbox *mb) -+{ -+ int status = bce_mailbox_retrive_response(mb); -+ if (!status) { -+ atomic_set(&mb->mb_status, 2); -+ complete(&mb->mb_completion); -+ } -+ return status; -+} -+ -+static void bc_send_timestamp(struct timer_list *tl); -+ -+void bce_timestamp_init(struct bce_timestamp *ts, void __iomem *reg) -+{ -+ u32 __iomem *regb; -+ -+ spin_lock_init(&ts->stop_sl); -+ ts->stopped = false; -+ -+ ts->reg = reg; -+ -+ regb = (u32*) ((u8*) ts->reg + REG_TIMESTAMP_BASE); -+ -+ ioread32(regb); -+ mb(); -+ -+ timer_setup(&ts->timer, bc_send_timestamp, 0); -+} -+ -+void bce_timestamp_start(struct bce_timestamp *ts, bool is_initial) -+{ -+ unsigned long flags; -+ u32 __iomem *regb = (u32*) ((u8*) ts->reg + REG_TIMESTAMP_BASE); -+ -+ if (is_initial) { -+ iowrite32((u32) -4, regb + 2); -+ iowrite32((u32) -1, regb); -+ } else { -+ iowrite32((u32) -3, regb + 2); -+ iowrite32((u32) -1, regb); -+ } -+ -+ spin_lock_irqsave(&ts->stop_sl, flags); -+ ts->stopped = false; -+ spin_unlock_irqrestore(&ts->stop_sl, flags); -+ mod_timer(&ts->timer, jiffies + msecs_to_jiffies(150)); -+} -+ -+void bce_timestamp_stop(struct bce_timestamp *ts) -+{ -+ unsigned long flags; -+ u32 __iomem *regb = (u32*) ((u8*) ts->reg + REG_TIMESTAMP_BASE); -+ -+ spin_lock_irqsave(&ts->stop_sl, flags); -+ ts->stopped = true; -+ spin_unlock_irqrestore(&ts->stop_sl, flags); -+ del_timer_sync(&ts->timer); -+ -+ iowrite32((u32) -2, regb + 2); -+ iowrite32((u32) -1, regb); -+} -+ -+static void bc_send_timestamp(struct timer_list *tl) -+{ -+ struct bce_timestamp *ts; -+ unsigned long flags; -+ u32 __iomem *regb; -+ ktime_t bt; -+ -+ ts = container_of(tl, struct bce_timestamp, timer); -+ regb = (u32*) ((u8*) ts->reg + REG_TIMESTAMP_BASE); -+ local_irq_save(flags); -+ ioread32(regb + 2); -+ mb(); -+ bt = ktime_get_boottime(); -+ iowrite32((u32) bt, regb + 2); -+ iowrite32((u32) (bt >> 32), regb); -+ -+ spin_lock(&ts->stop_sl); -+ if (!ts->stopped) -+ mod_timer(&ts->timer, jiffies + msecs_to_jiffies(150)); -+ spin_unlock(&ts->stop_sl); -+ local_irq_restore(flags); -+} -\ No newline at end of file -diff --git a/drivers/staging/apple-bce/mailbox.h b/drivers/staging/apple-bce/mailbox.h -new file mode 100644 -index 000000000000..f3323f95ba51 ---- /dev/null -+++ b/drivers/staging/apple-bce/mailbox.h -@@ -0,0 +1,53 @@ -+#ifndef BCE_MAILBOX_H -+#define BCE_MAILBOX_H -+ -+#include -+#include -+#include -+ -+struct bce_mailbox { -+ void __iomem *reg_mb; -+ -+ atomic_t mb_status; // possible statuses: 0 (no msg), 1 (has active msg), 2 (got reply) -+ struct completion mb_completion; -+ uint64_t mb_result; -+}; -+ -+enum bce_message_type { -+ BCE_MB_REGISTER_COMMAND_SQ = 0x7, // to-device -+ BCE_MB_REGISTER_COMMAND_CQ = 0x8, // to-device -+ BCE_MB_REGISTER_COMMAND_QUEUE_REPLY = 0xB, // to-host -+ BCE_MB_SET_FW_PROTOCOL_VERSION = 0xC, // both -+ BCE_MB_SLEEP_NO_STATE = 0x14, // to-device -+ BCE_MB_RESTORE_NO_STATE = 0x15, // to-device -+ BCE_MB_SAVE_STATE_AND_SLEEP = 0x17, // to-device -+ BCE_MB_RESTORE_STATE_AND_WAKE = 0x18, // to-device -+ BCE_MB_SAVE_STATE_AND_SLEEP_FAILURE = 0x19, // from-device -+ BCE_MB_SAVE_RESTORE_STATE_COMPLETE = 0x1A, // from-device -+}; -+ -+#define BCE_MB_MSG(type, value) (((u64) (type) << 58) | ((value) & 0x3FFFFFFFFFFFFFFLL)) -+#define BCE_MB_TYPE(v) ((u32) (v >> 58)) -+#define BCE_MB_VALUE(v) (v & 0x3FFFFFFFFFFFFFFLL) -+ -+void bce_mailbox_init(struct bce_mailbox *mb, void __iomem *reg_mb); -+ -+int bce_mailbox_send(struct bce_mailbox *mb, u64 msg, u64* recv); -+ -+int bce_mailbox_handle_interrupt(struct bce_mailbox *mb); -+ -+ -+struct bce_timestamp { -+ void __iomem *reg; -+ struct timer_list timer; -+ struct spinlock stop_sl; -+ bool stopped; -+}; -+ -+void bce_timestamp_init(struct bce_timestamp *ts, void __iomem *reg); -+ -+void bce_timestamp_start(struct bce_timestamp *ts, bool is_initial); -+ -+void bce_timestamp_stop(struct bce_timestamp *ts); -+ -+#endif //BCEDRIVER_MAILBOX_H -diff --git a/drivers/staging/apple-bce/queue.c b/drivers/staging/apple-bce/queue.c -new file mode 100644 -index 000000000000..bc9cd3bc6f0c ---- /dev/null -+++ b/drivers/staging/apple-bce/queue.c -@@ -0,0 +1,390 @@ -+#include "queue.h" -+#include "apple_bce.h" -+ -+#define REG_DOORBELL_BASE 0x44000 -+ -+struct bce_queue_cq *bce_alloc_cq(struct apple_bce_device *dev, int qid, u32 el_count) -+{ -+ struct bce_queue_cq *q; -+ q = kzalloc(sizeof(struct bce_queue_cq), GFP_KERNEL); -+ q->qid = qid; -+ q->type = BCE_QUEUE_CQ; -+ q->el_count = el_count; -+ q->data = dma_alloc_coherent(&dev->pci->dev, el_count * sizeof(struct bce_qe_completion), -+ &q->dma_handle, GFP_KERNEL); -+ if (!q->data) { -+ pr_err("DMA queue memory alloc failed\n"); -+ kfree(q); -+ return NULL; -+ } -+ return q; -+} -+ -+void bce_get_cq_memcfg(struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg) -+{ -+ cfg->qid = (u16) cq->qid; -+ cfg->el_count = (u16) cq->el_count; -+ cfg->vector_or_cq = 0; -+ cfg->_pad = 0; -+ cfg->addr = cq->dma_handle; -+ cfg->length = cq->el_count * sizeof(struct bce_qe_completion); -+} -+ -+void bce_free_cq(struct apple_bce_device *dev, struct bce_queue_cq *cq) -+{ -+ dma_free_coherent(&dev->pci->dev, cq->el_count * sizeof(struct bce_qe_completion), cq->data, cq->dma_handle); -+ kfree(cq); -+} -+ -+static void bce_handle_cq_completion(struct apple_bce_device *dev, struct bce_qe_completion *e, size_t *ce) -+{ -+ struct bce_queue *target; -+ struct bce_queue_sq *target_sq; -+ struct bce_sq_completion_data *cmpl; -+ if (e->qid >= BCE_MAX_QUEUE_COUNT) { -+ pr_err("Device sent a response for qid (%u) >= BCE_MAX_QUEUE_COUNT\n", e->qid); -+ return; -+ } -+ target = dev->queues[e->qid]; -+ if (!target || target->type != BCE_QUEUE_SQ) { -+ pr_err("Device sent a response for qid (%u), which does not exist\n", e->qid); -+ return; -+ } -+ target_sq = (struct bce_queue_sq *) target; -+ if (target_sq->completion_tail != e->completion_index) { -+ pr_err("Completion index mismatch; this is likely going to make this driver unusable\n"); -+ return; -+ } -+ if (!target_sq->has_pending_completions) { -+ target_sq->has_pending_completions = true; -+ dev->int_sq_list[(*ce)++] = target_sq; -+ } -+ cmpl = &target_sq->completion_data[e->completion_index]; -+ cmpl->status = e->status; -+ cmpl->data_size = e->data_size; -+ cmpl->result = e->result; -+ wmb(); -+ target_sq->completion_tail = (target_sq->completion_tail + 1) % target_sq->el_count; -+} -+ -+void bce_handle_cq_completions(struct apple_bce_device *dev, struct bce_queue_cq *cq) -+{ -+ size_t ce = 0; -+ struct bce_qe_completion *e; -+ struct bce_queue_sq *sq; -+ e = bce_cq_element(cq, cq->index); -+ if (!(e->flags & BCE_COMPLETION_FLAG_PENDING)) -+ return; -+ mb(); -+ while (true) { -+ e = bce_cq_element(cq, cq->index); -+ if (!(e->flags & BCE_COMPLETION_FLAG_PENDING)) -+ break; -+ // pr_info("apple-bce: compl: %i: %i %llx %llx", e->qid, e->status, e->data_size, e->result); -+ bce_handle_cq_completion(dev, e, &ce); -+ e->flags = 0; -+ cq->index = (cq->index + 1) % cq->el_count; -+ } -+ mb(); -+ iowrite32(cq->index, (u32 *) ((u8 *) dev->reg_mem_dma + REG_DOORBELL_BASE) + cq->qid); -+ while (ce) { -+ --ce; -+ sq = dev->int_sq_list[ce]; -+ sq->completion(sq); -+ sq->has_pending_completions = false; -+ } -+} -+ -+ -+struct bce_queue_sq *bce_alloc_sq(struct apple_bce_device *dev, int qid, u32 el_size, u32 el_count, -+ bce_sq_completion compl, void *userdata) -+{ -+ struct bce_queue_sq *q; -+ q = kzalloc(sizeof(struct bce_queue_sq), GFP_KERNEL); -+ q->qid = qid; -+ q->type = BCE_QUEUE_SQ; -+ q->el_size = el_size; -+ q->el_count = el_count; -+ q->data = dma_alloc_coherent(&dev->pci->dev, el_count * el_size, -+ &q->dma_handle, GFP_KERNEL); -+ q->completion = compl; -+ q->userdata = userdata; -+ q->completion_data = kzalloc(sizeof(struct bce_sq_completion_data) * el_count, GFP_KERNEL); -+ q->reg_mem_dma = dev->reg_mem_dma; -+ atomic_set(&q->available_commands, el_count - 1); -+ init_completion(&q->available_command_completion); -+ atomic_set(&q->available_command_completion_waiting_count, 0); -+ if (!q->data) { -+ pr_err("DMA queue memory alloc failed\n"); -+ kfree(q); -+ return NULL; -+ } -+ return q; -+} -+ -+void bce_get_sq_memcfg(struct bce_queue_sq *sq, struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg) -+{ -+ cfg->qid = (u16) sq->qid; -+ cfg->el_count = (u16) sq->el_count; -+ cfg->vector_or_cq = (u16) cq->qid; -+ cfg->_pad = 0; -+ cfg->addr = sq->dma_handle; -+ cfg->length = sq->el_count * sq->el_size; -+} -+ -+void bce_free_sq(struct apple_bce_device *dev, struct bce_queue_sq *sq) -+{ -+ dma_free_coherent(&dev->pci->dev, sq->el_count * sq->el_size, sq->data, sq->dma_handle); -+ kfree(sq); -+} -+ -+int bce_reserve_submission(struct bce_queue_sq *sq, unsigned long *timeout) -+{ -+ while (atomic_dec_if_positive(&sq->available_commands) < 0) { -+ if (!timeout || !*timeout) -+ return -EAGAIN; -+ atomic_inc(&sq->available_command_completion_waiting_count); -+ *timeout = wait_for_completion_timeout(&sq->available_command_completion, *timeout); -+ if (!*timeout) { -+ if (atomic_dec_if_positive(&sq->available_command_completion_waiting_count) < 0) -+ try_wait_for_completion(&sq->available_command_completion); /* consume the pending completion */ -+ } -+ } -+ return 0; -+} -+ -+void bce_cancel_submission_reservation(struct bce_queue_sq *sq) -+{ -+ atomic_inc(&sq->available_commands); -+} -+ -+void *bce_next_submission(struct bce_queue_sq *sq) -+{ -+ void *ret = bce_sq_element(sq, sq->tail); -+ sq->tail = (sq->tail + 1) % sq->el_count; -+ return ret; -+} -+ -+void bce_submit_to_device(struct bce_queue_sq *sq) -+{ -+ mb(); -+ iowrite32(sq->tail, (u32 *) ((u8 *) sq->reg_mem_dma + REG_DOORBELL_BASE) + sq->qid); -+} -+ -+void bce_notify_submission_complete(struct bce_queue_sq *sq) -+{ -+ sq->head = (sq->head + 1) % sq->el_count; -+ atomic_inc(&sq->available_commands); -+ if (atomic_dec_if_positive(&sq->available_command_completion_waiting_count) >= 0) { -+ complete(&sq->available_command_completion); -+ } -+} -+ -+void bce_set_submission_single(struct bce_qe_submission *element, dma_addr_t addr, size_t size) -+{ -+ element->addr = addr; -+ element->length = size; -+ element->segl_addr = element->segl_length = 0; -+} -+ -+static void bce_cmdq_completion(struct bce_queue_sq *q); -+ -+struct bce_queue_cmdq *bce_alloc_cmdq(struct apple_bce_device *dev, int qid, u32 el_count) -+{ -+ struct bce_queue_cmdq *q; -+ q = kzalloc(sizeof(struct bce_queue_cmdq), GFP_KERNEL); -+ q->sq = bce_alloc_sq(dev, qid, BCE_CMD_SIZE, el_count, bce_cmdq_completion, q); -+ if (!q->sq) { -+ kfree(q); -+ return NULL; -+ } -+ spin_lock_init(&q->lck); -+ q->tres = kzalloc(sizeof(struct bce_queue_cmdq_result_el*) * el_count, GFP_KERNEL); -+ if (!q->tres) { -+ kfree(q); -+ return NULL; -+ } -+ return q; -+} -+ -+void bce_free_cmdq(struct apple_bce_device *dev, struct bce_queue_cmdq *cmdq) -+{ -+ bce_free_sq(dev, cmdq->sq); -+ kfree(cmdq->tres); -+ kfree(cmdq); -+} -+ -+void bce_cmdq_completion(struct bce_queue_sq *q) -+{ -+ struct bce_queue_cmdq_result_el *el; -+ struct bce_queue_cmdq *cmdq = q->userdata; -+ struct bce_sq_completion_data *result; -+ -+ spin_lock(&cmdq->lck); -+ while ((result = bce_next_completion(q))) { -+ el = cmdq->tres[cmdq->sq->head]; -+ if (el) { -+ el->result = result->result; -+ el->status = result->status; -+ mb(); -+ complete(&el->cmpl); -+ } else { -+ pr_err("apple-bce: Unexpected command queue completion\n"); -+ } -+ cmdq->tres[cmdq->sq->head] = NULL; -+ bce_notify_submission_complete(q); -+ } -+ spin_unlock(&cmdq->lck); -+} -+ -+static __always_inline void *bce_cmd_start(struct bce_queue_cmdq *cmdq, struct bce_queue_cmdq_result_el *res) -+{ -+ void *ret; -+ unsigned long timeout; -+ init_completion(&res->cmpl); -+ mb(); -+ -+ timeout = msecs_to_jiffies(1000L * 60 * 5); /* wait for up to ~5 minutes */ -+ if (bce_reserve_submission(cmdq->sq, &timeout)) -+ return NULL; -+ -+ spin_lock(&cmdq->lck); -+ cmdq->tres[cmdq->sq->tail] = res; -+ ret = bce_next_submission(cmdq->sq); -+ return ret; -+} -+ -+static __always_inline void bce_cmd_finish(struct bce_queue_cmdq *cmdq, struct bce_queue_cmdq_result_el *res) -+{ -+ bce_submit_to_device(cmdq->sq); -+ spin_unlock(&cmdq->lck); -+ -+ wait_for_completion(&res->cmpl); -+ mb(); -+} -+ -+u32 bce_cmd_register_queue(struct bce_queue_cmdq *cmdq, struct bce_queue_memcfg *cfg, const char *name, bool isdirout) -+{ -+ struct bce_queue_cmdq_result_el res; -+ struct bce_cmdq_register_memory_queue_cmd *cmd = bce_cmd_start(cmdq, &res); -+ if (!cmd) -+ return (u32) -1; -+ cmd->cmd = BCE_CMD_REGISTER_MEMORY_QUEUE; -+ cmd->flags = (u16) ((name ? 2 : 0) | (isdirout ? 1 : 0)); -+ cmd->qid = cfg->qid; -+ cmd->el_count = cfg->el_count; -+ cmd->vector_or_cq = cfg->vector_or_cq; -+ memset(cmd->name, 0, sizeof(cmd->name)); -+ if (name) { -+ cmd->name_len = (u16) min(strlen(name), (size_t) sizeof(cmd->name)); -+ memcpy(cmd->name, name, cmd->name_len); -+ } else { -+ cmd->name_len = 0; -+ } -+ cmd->addr = cfg->addr; -+ cmd->length = cfg->length; -+ -+ bce_cmd_finish(cmdq, &res); -+ return res.status; -+} -+ -+u32 bce_cmd_unregister_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid) -+{ -+ struct bce_queue_cmdq_result_el res; -+ struct bce_cmdq_simple_memory_queue_cmd *cmd = bce_cmd_start(cmdq, &res); -+ if (!cmd) -+ return (u32) -1; -+ cmd->cmd = BCE_CMD_UNREGISTER_MEMORY_QUEUE; -+ cmd->flags = 0; -+ cmd->qid = qid; -+ bce_cmd_finish(cmdq, &res); -+ return res.status; -+} -+ -+u32 bce_cmd_flush_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid) -+{ -+ struct bce_queue_cmdq_result_el res; -+ struct bce_cmdq_simple_memory_queue_cmd *cmd = bce_cmd_start(cmdq, &res); -+ if (!cmd) -+ return (u32) -1; -+ cmd->cmd = BCE_CMD_FLUSH_MEMORY_QUEUE; -+ cmd->flags = 0; -+ cmd->qid = qid; -+ bce_cmd_finish(cmdq, &res); -+ return res.status; -+} -+ -+ -+struct bce_queue_cq *bce_create_cq(struct apple_bce_device *dev, u32 el_count) -+{ -+ struct bce_queue_cq *cq; -+ struct bce_queue_memcfg cfg; -+ int qid = ida_simple_get(&dev->queue_ida, BCE_QUEUE_USER_MIN, BCE_QUEUE_USER_MAX, GFP_KERNEL); -+ if (qid < 0) -+ return NULL; -+ cq = bce_alloc_cq(dev, qid, el_count); -+ if (!cq) -+ return NULL; -+ bce_get_cq_memcfg(cq, &cfg); -+ if (bce_cmd_register_queue(dev->cmd_cmdq, &cfg, NULL, false) != 0) { -+ pr_err("apple-bce: CQ registration failed (%i)", qid); -+ bce_free_cq(dev, cq); -+ ida_simple_remove(&dev->queue_ida, (uint) qid); -+ return NULL; -+ } -+ dev->queues[qid] = (struct bce_queue *) cq; -+ return cq; -+} -+ -+struct bce_queue_sq *bce_create_sq(struct apple_bce_device *dev, struct bce_queue_cq *cq, const char *name, u32 el_count, -+ int direction, bce_sq_completion compl, void *userdata) -+{ -+ struct bce_queue_sq *sq; -+ struct bce_queue_memcfg cfg; -+ int qid; -+ if (cq == NULL) -+ return NULL; /* cq can not be null */ -+ if (name == NULL) -+ return NULL; /* name can not be null */ -+ if (direction != DMA_TO_DEVICE && direction != DMA_FROM_DEVICE) -+ return NULL; /* unsupported direction */ -+ qid = ida_simple_get(&dev->queue_ida, BCE_QUEUE_USER_MIN, BCE_QUEUE_USER_MAX, GFP_KERNEL); -+ if (qid < 0) -+ return NULL; -+ sq = bce_alloc_sq(dev, qid, sizeof(struct bce_qe_submission), el_count, compl, userdata); -+ if (!sq) -+ return NULL; -+ bce_get_sq_memcfg(sq, cq, &cfg); -+ if (bce_cmd_register_queue(dev->cmd_cmdq, &cfg, name, direction != DMA_FROM_DEVICE) != 0) { -+ pr_err("apple-bce: SQ registration failed (%i)", qid); -+ bce_free_sq(dev, sq); -+ ida_simple_remove(&dev->queue_ida, (uint) qid); -+ return NULL; -+ } -+ spin_lock(&dev->queues_lock); -+ dev->queues[qid] = (struct bce_queue *) sq; -+ spin_unlock(&dev->queues_lock); -+ return sq; -+} -+ -+void bce_destroy_cq(struct apple_bce_device *dev, struct bce_queue_cq *cq) -+{ -+ if (!dev->is_being_removed && bce_cmd_unregister_memory_queue(dev->cmd_cmdq, (u16) cq->qid)) -+ pr_err("apple-bce: CQ unregister failed"); -+ spin_lock(&dev->queues_lock); -+ dev->queues[cq->qid] = NULL; -+ spin_unlock(&dev->queues_lock); -+ ida_simple_remove(&dev->queue_ida, (uint) cq->qid); -+ bce_free_cq(dev, cq); -+} -+ -+void bce_destroy_sq(struct apple_bce_device *dev, struct bce_queue_sq *sq) -+{ -+ if (!dev->is_being_removed && bce_cmd_unregister_memory_queue(dev->cmd_cmdq, (u16) sq->qid)) -+ pr_err("apple-bce: CQ unregister failed"); -+ spin_lock(&dev->queues_lock); -+ dev->queues[sq->qid] = NULL; -+ spin_unlock(&dev->queues_lock); -+ ida_simple_remove(&dev->queue_ida, (uint) sq->qid); -+ bce_free_sq(dev, sq); -+} -\ No newline at end of file -diff --git a/drivers/staging/apple-bce/queue.h b/drivers/staging/apple-bce/queue.h -new file mode 100644 -index 000000000000..8368ac5dfca8 ---- /dev/null -+++ b/drivers/staging/apple-bce/queue.h -@@ -0,0 +1,177 @@ -+#ifndef BCE_QUEUE_H -+#define BCE_QUEUE_H -+ -+#include -+#include -+ -+#define BCE_CMD_SIZE 0x40 -+ -+struct apple_bce_device; -+ -+enum bce_queue_type { -+ BCE_QUEUE_CQ, BCE_QUEUE_SQ -+}; -+struct bce_queue { -+ int qid; -+ int type; -+}; -+struct bce_queue_cq { -+ int qid; -+ int type; -+ u32 el_count; -+ dma_addr_t dma_handle; -+ void *data; -+ -+ u32 index; -+}; -+struct bce_queue_sq; -+typedef void (*bce_sq_completion)(struct bce_queue_sq *q); -+struct bce_sq_completion_data { -+ u32 status; -+ u64 data_size; -+ u64 result; -+}; -+struct bce_queue_sq { -+ int qid; -+ int type; -+ u32 el_size; -+ u32 el_count; -+ dma_addr_t dma_handle; -+ void *data; -+ void *userdata; -+ void __iomem *reg_mem_dma; -+ -+ atomic_t available_commands; -+ struct completion available_command_completion; -+ atomic_t available_command_completion_waiting_count; -+ u32 head, tail; -+ -+ u32 completion_cidx, completion_tail; -+ struct bce_sq_completion_data *completion_data; -+ bool has_pending_completions; -+ bce_sq_completion completion; -+}; -+ -+struct bce_queue_cmdq_result_el { -+ struct completion cmpl; -+ u32 status; -+ u64 result; -+}; -+struct bce_queue_cmdq { -+ struct bce_queue_sq *sq; -+ struct spinlock lck; -+ struct bce_queue_cmdq_result_el **tres; -+}; -+ -+struct bce_queue_memcfg { -+ u16 qid; -+ u16 el_count; -+ u16 vector_or_cq; -+ u16 _pad; -+ u64 addr; -+ u64 length; -+}; -+ -+enum bce_qe_completion_status { -+ BCE_COMPLETION_SUCCESS = 0, -+ BCE_COMPLETION_ERROR = 1, -+ BCE_COMPLETION_ABORTED = 2, -+ BCE_COMPLETION_NO_SPACE = 3, -+ BCE_COMPLETION_OVERRUN = 4 -+}; -+enum bce_qe_completion_flags { -+ BCE_COMPLETION_FLAG_PENDING = 0x8000 -+}; -+struct bce_qe_completion { -+ u64 result; -+ u64 data_size; -+ u16 qid; -+ u16 completion_index; -+ u16 status; // bce_qe_completion_status -+ u16 flags; // bce_qe_completion_flags -+}; -+ -+struct bce_qe_submission { -+ u64 length; -+ u64 addr; -+ -+ u64 segl_addr; -+ u64 segl_length; -+}; -+ -+enum bce_cmdq_command { -+ BCE_CMD_REGISTER_MEMORY_QUEUE = 0x20, -+ BCE_CMD_UNREGISTER_MEMORY_QUEUE = 0x30, -+ BCE_CMD_FLUSH_MEMORY_QUEUE = 0x40, -+ BCE_CMD_SET_MEMORY_QUEUE_PROPERTY = 0x50 -+}; -+struct bce_cmdq_simple_memory_queue_cmd { -+ u16 cmd; // bce_cmdq_command -+ u16 flags; -+ u16 qid; -+}; -+struct bce_cmdq_register_memory_queue_cmd { -+ u16 cmd; // bce_cmdq_command -+ u16 flags; -+ u16 qid; -+ u16 _pad; -+ u16 el_count; -+ u16 vector_or_cq; -+ u16 _pad2; -+ u16 name_len; -+ char name[0x20]; -+ u64 addr; -+ u64 length; -+}; -+ -+static __always_inline void *bce_sq_element(struct bce_queue_sq *q, int i) { -+ return (void *) ((u8 *) q->data + q->el_size * i); -+} -+static __always_inline void *bce_cq_element(struct bce_queue_cq *q, int i) { -+ return (void *) ((struct bce_qe_completion *) q->data + i); -+} -+ -+static __always_inline struct bce_sq_completion_data *bce_next_completion(struct bce_queue_sq *sq) { -+ struct bce_sq_completion_data *res; -+ rmb(); -+ if (sq->completion_cidx == sq->completion_tail) -+ return NULL; -+ res = &sq->completion_data[sq->completion_cidx]; -+ sq->completion_cidx = (sq->completion_cidx + 1) % sq->el_count; -+ return res; -+} -+ -+struct bce_queue_cq *bce_alloc_cq(struct apple_bce_device *dev, int qid, u32 el_count); -+void bce_get_cq_memcfg(struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg); -+void bce_free_cq(struct apple_bce_device *dev, struct bce_queue_cq *cq); -+void bce_handle_cq_completions(struct apple_bce_device *dev, struct bce_queue_cq *cq); -+ -+struct bce_queue_sq *bce_alloc_sq(struct apple_bce_device *dev, int qid, u32 el_size, u32 el_count, -+ bce_sq_completion compl, void *userdata); -+void bce_get_sq_memcfg(struct bce_queue_sq *sq, struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg); -+void bce_free_sq(struct apple_bce_device *dev, struct bce_queue_sq *sq); -+int bce_reserve_submission(struct bce_queue_sq *sq, unsigned long *timeout); -+void bce_cancel_submission_reservation(struct bce_queue_sq *sq); -+void *bce_next_submission(struct bce_queue_sq *sq); -+void bce_submit_to_device(struct bce_queue_sq *sq); -+void bce_notify_submission_complete(struct bce_queue_sq *sq); -+ -+void bce_set_submission_single(struct bce_qe_submission *element, dma_addr_t addr, size_t size); -+ -+struct bce_queue_cmdq *bce_alloc_cmdq(struct apple_bce_device *dev, int qid, u32 el_count); -+void bce_free_cmdq(struct apple_bce_device *dev, struct bce_queue_cmdq *cmdq); -+ -+u32 bce_cmd_register_queue(struct bce_queue_cmdq *cmdq, struct bce_queue_memcfg *cfg, const char *name, bool isdirout); -+u32 bce_cmd_unregister_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid); -+u32 bce_cmd_flush_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid); -+ -+ -+/* User API - Creates and registers the queue */ -+ -+struct bce_queue_cq *bce_create_cq(struct apple_bce_device *dev, u32 el_count); -+struct bce_queue_sq *bce_create_sq(struct apple_bce_device *dev, struct bce_queue_cq *cq, const char *name, u32 el_count, -+ int direction, bce_sq_completion compl, void *userdata); -+void bce_destroy_cq(struct apple_bce_device *dev, struct bce_queue_cq *cq); -+void bce_destroy_sq(struct apple_bce_device *dev, struct bce_queue_sq *sq); -+ -+#endif //BCEDRIVER_MAILBOX_H -diff --git a/drivers/staging/apple-bce/queue_dma.c b/drivers/staging/apple-bce/queue_dma.c -new file mode 100644 -index 000000000000..b236613285c0 ---- /dev/null -+++ b/drivers/staging/apple-bce/queue_dma.c -@@ -0,0 +1,220 @@ -+#include "queue_dma.h" -+#include -+#include -+#include "queue.h" -+ -+static int bce_alloc_scatterlist_from_vm(struct sg_table *tbl, void *data, size_t len); -+static struct bce_segment_list_element_hostinfo *bce_map_segment_list( -+ struct device *dev, struct scatterlist *pages, int pagen); -+static void bce_unmap_segement_list(struct device *dev, struct bce_segment_list_element_hostinfo *list); -+ -+int bce_map_dma_buffer(struct device *dev, struct bce_dma_buffer *buf, struct sg_table scatterlist, -+ enum dma_data_direction dir) -+{ -+ int cnt; -+ -+ buf->direction = dir; -+ buf->scatterlist = scatterlist; -+ buf->seglist_hostinfo = NULL; -+ -+ cnt = dma_map_sg(dev, buf->scatterlist.sgl, buf->scatterlist.nents, dir); -+ if (cnt != buf->scatterlist.nents) { -+ pr_err("apple-bce: DMA scatter list mapping returned an unexpected count: %i\n", cnt); -+ dma_unmap_sg(dev, buf->scatterlist.sgl, buf->scatterlist.nents, dir); -+ return -EIO; -+ } -+ if (cnt == 1) -+ return 0; -+ -+ buf->seglist_hostinfo = bce_map_segment_list(dev, buf->scatterlist.sgl, buf->scatterlist.nents); -+ if (!buf->seglist_hostinfo) { -+ pr_err("apple-bce: Creating segment list failed\n"); -+ dma_unmap_sg(dev, buf->scatterlist.sgl, buf->scatterlist.nents, dir); -+ return -EIO; -+ } -+ return 0; -+} -+ -+int bce_map_dma_buffer_vm(struct device *dev, struct bce_dma_buffer *buf, void *data, size_t len, -+ enum dma_data_direction dir) -+{ -+ int status; -+ struct sg_table scatterlist; -+ if ((status = bce_alloc_scatterlist_from_vm(&scatterlist, data, len))) -+ return status; -+ if ((status = bce_map_dma_buffer(dev, buf, scatterlist, dir))) { -+ sg_free_table(&scatterlist); -+ return status; -+ } -+ return 0; -+} -+ -+int bce_map_dma_buffer_km(struct device *dev, struct bce_dma_buffer *buf, void *data, size_t len, -+ enum dma_data_direction dir) -+{ -+ /* Kernel memory is continuous which is great for us. */ -+ int status; -+ struct sg_table scatterlist; -+ if ((status = sg_alloc_table(&scatterlist, 1, GFP_KERNEL))) { -+ sg_free_table(&scatterlist); -+ return status; -+ } -+ sg_set_buf(scatterlist.sgl, data, (uint) len); -+ if ((status = bce_map_dma_buffer(dev, buf, scatterlist, dir))) { -+ sg_free_table(&scatterlist); -+ return status; -+ } -+ return 0; -+} -+ -+void bce_unmap_dma_buffer(struct device *dev, struct bce_dma_buffer *buf) -+{ -+ dma_unmap_sg(dev, buf->scatterlist.sgl, buf->scatterlist.nents, buf->direction); -+ bce_unmap_segement_list(dev, buf->seglist_hostinfo); -+} -+ -+ -+static int bce_alloc_scatterlist_from_vm(struct sg_table *tbl, void *data, size_t len) -+{ -+ int status, i; -+ struct page **pages; -+ size_t off, start_page, end_page, page_count; -+ off = (size_t) data % PAGE_SIZE; -+ start_page = (size_t) data / PAGE_SIZE; -+ end_page = ((size_t) data + len - 1) / PAGE_SIZE; -+ page_count = end_page - start_page + 1; -+ -+ if (page_count > PAGE_SIZE / sizeof(struct page *)) -+ pages = vmalloc(page_count * sizeof(struct page *)); -+ else -+ pages = kmalloc(page_count * sizeof(struct page *), GFP_KERNEL); -+ -+ for (i = 0; i < page_count; i++) -+ pages[i] = vmalloc_to_page((void *) ((start_page + i) * PAGE_SIZE)); -+ -+ if ((status = sg_alloc_table_from_pages(tbl, pages, page_count, (unsigned int) off, len, GFP_KERNEL))) { -+ sg_free_table(tbl); -+ } -+ -+ if (page_count > PAGE_SIZE / sizeof(struct page *)) -+ vfree(pages); -+ else -+ kfree(pages); -+ return status; -+} -+ -+#define BCE_ELEMENTS_PER_PAGE ((PAGE_SIZE - sizeof(struct bce_segment_list_header)) \ -+ / sizeof(struct bce_segment_list_element)) -+#define BCE_ELEMENTS_PER_ADDITIONAL_PAGE (PAGE_SIZE / sizeof(struct bce_segment_list_element)) -+ -+static struct bce_segment_list_element_hostinfo *bce_map_segment_list( -+ struct device *dev, struct scatterlist *pages, int pagen) -+{ -+ size_t ptr, pptr = 0; -+ struct bce_segment_list_header theader; /* a temp header, to store the initial seg */ -+ struct bce_segment_list_header *header; -+ struct bce_segment_list_element *el, *el_end; -+ struct bce_segment_list_element_hostinfo *out, *pout, *out_root; -+ struct scatterlist *sg; -+ int i; -+ header = &theader; -+ out = out_root = NULL; -+ el = el_end = NULL; -+ for_each_sg(pages, sg, pagen, i) { -+ if (el >= el_end) { -+ /* allocate a new page, this will be also done for the first element */ -+ ptr = __get_free_page(GFP_KERNEL); -+ if (pptr && ptr == pptr + PAGE_SIZE) { -+ out->page_count++; -+ header->element_count += BCE_ELEMENTS_PER_ADDITIONAL_PAGE; -+ el_end += BCE_ELEMENTS_PER_ADDITIONAL_PAGE; -+ } else { -+ header = (void *) ptr; -+ header->element_count = BCE_ELEMENTS_PER_PAGE; -+ header->data_size = 0; -+ header->next_segl_addr = 0; -+ header->next_segl_length = 0; -+ el = (void *) (header + 1); -+ el_end = el + BCE_ELEMENTS_PER_PAGE; -+ -+ if (out) { -+ out->next = kmalloc(sizeof(struct bce_segment_list_element_hostinfo), GFP_KERNEL); -+ out = out->next; -+ } else { -+ out_root = out = kmalloc(sizeof(struct bce_segment_list_element_hostinfo), GFP_KERNEL); -+ } -+ out->page_start = (void *) ptr; -+ out->page_count = 1; -+ out->dma_start = DMA_MAPPING_ERROR; -+ out->next = NULL; -+ } -+ pptr = ptr; -+ } -+ el->addr = sg->dma_address; -+ el->length = sg->length; -+ header->data_size += el->length; -+ } -+ -+ /* DMA map */ -+ out = out_root; -+ pout = NULL; -+ while (out) { -+ out->dma_start = dma_map_single(dev, out->page_start, out->page_count * PAGE_SIZE, DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, out->dma_start)) -+ goto error; -+ if (pout) { -+ header = pout->page_start; -+ header->next_segl_addr = out->dma_start; -+ header->next_segl_length = out->page_count * PAGE_SIZE; -+ } -+ pout = out; -+ out = out->next; -+ } -+ return out_root; -+ -+ error: -+ bce_unmap_segement_list(dev, out_root); -+ return NULL; -+} -+ -+static void bce_unmap_segement_list(struct device *dev, struct bce_segment_list_element_hostinfo *list) -+{ -+ struct bce_segment_list_element_hostinfo *next; -+ while (list) { -+ if (list->dma_start != DMA_MAPPING_ERROR) -+ dma_unmap_single(dev, list->dma_start, list->page_count * PAGE_SIZE, DMA_TO_DEVICE); -+ next = list->next; -+ kfree(list); -+ list = next; -+ } -+} -+ -+int bce_set_submission_buf(struct bce_qe_submission *element, struct bce_dma_buffer *buf, size_t offset, size_t length) -+{ -+ struct bce_segment_list_element_hostinfo *seg; -+ struct bce_segment_list_header *seg_header; -+ -+ seg = buf->seglist_hostinfo; -+ if (!seg) { -+ element->addr = buf->scatterlist.sgl->dma_address + offset; -+ element->length = length; -+ element->segl_addr = 0; -+ element->segl_length = 0; -+ return 0; -+ } -+ -+ while (seg) { -+ seg_header = seg->page_start; -+ if (offset <= seg_header->data_size) -+ break; -+ offset -= seg_header->data_size; -+ seg = seg->next; -+ } -+ if (!seg) -+ return -EINVAL; -+ element->addr = offset; -+ element->length = buf->scatterlist.sgl->dma_length; -+ element->segl_addr = seg->dma_start; -+ element->segl_length = seg->page_count * PAGE_SIZE; -+ return 0; -+} -\ No newline at end of file -diff --git a/drivers/staging/apple-bce/queue_dma.h b/drivers/staging/apple-bce/queue_dma.h -new file mode 100644 -index 000000000000..f8a57e50e7a3 ---- /dev/null -+++ b/drivers/staging/apple-bce/queue_dma.h -@@ -0,0 +1,50 @@ -+#ifndef BCE_QUEUE_DMA_H -+#define BCE_QUEUE_DMA_H -+ -+#include -+ -+struct bce_qe_submission; -+ -+struct bce_segment_list_header { -+ u64 element_count; -+ u64 data_size; -+ -+ u64 next_segl_addr; -+ u64 next_segl_length; -+}; -+struct bce_segment_list_element { -+ u64 addr; -+ u64 length; -+}; -+ -+struct bce_segment_list_element_hostinfo { -+ struct bce_segment_list_element_hostinfo *next; -+ void *page_start; -+ size_t page_count; -+ dma_addr_t dma_start; -+}; -+ -+ -+struct bce_dma_buffer { -+ enum dma_data_direction direction; -+ struct sg_table scatterlist; -+ struct bce_segment_list_element_hostinfo *seglist_hostinfo; -+}; -+ -+/* NOTE: Takes ownership of the sg_table if it succeeds. Ownership is not transferred on failure. */ -+int bce_map_dma_buffer(struct device *dev, struct bce_dma_buffer *buf, struct sg_table scatterlist, -+ enum dma_data_direction dir); -+ -+/* Creates a buffer from virtual memory (vmalloc) */ -+int bce_map_dma_buffer_vm(struct device *dev, struct bce_dma_buffer *buf, void *data, size_t len, -+ enum dma_data_direction dir); -+ -+/* Creates a buffer from kernel memory (kmalloc) */ -+int bce_map_dma_buffer_km(struct device *dev, struct bce_dma_buffer *buf, void *data, size_t len, -+ enum dma_data_direction dir); -+ -+void bce_unmap_dma_buffer(struct device *dev, struct bce_dma_buffer *buf); -+ -+int bce_set_submission_buf(struct bce_qe_submission *element, struct bce_dma_buffer *buf, size_t offset, size_t length); -+ -+#endif //BCE_QUEUE_DMA_H -diff --git a/drivers/staging/apple-bce/vhci/command.h b/drivers/staging/apple-bce/vhci/command.h -new file mode 100644 -index 000000000000..26619e0bccfa ---- /dev/null -+++ b/drivers/staging/apple-bce/vhci/command.h -@@ -0,0 +1,204 @@ -+#ifndef BCE_VHCI_COMMAND_H -+#define BCE_VHCI_COMMAND_H -+ -+#include "queue.h" -+#include -+#include -+ -+#define BCE_VHCI_CMD_TIMEOUT_SHORT msecs_to_jiffies(2000) -+#define BCE_VHCI_CMD_TIMEOUT_LONG msecs_to_jiffies(30000) -+ -+#define BCE_VHCI_BULK_MAX_ACTIVE_URBS_POW2 2 -+#define BCE_VHCI_BULK_MAX_ACTIVE_URBS (1 << BCE_VHCI_BULK_MAX_ACTIVE_URBS_POW2) -+ -+typedef u8 bce_vhci_port_t; -+typedef u8 bce_vhci_device_t; -+ -+enum bce_vhci_command { -+ BCE_VHCI_CMD_CONTROLLER_ENABLE = 1, -+ BCE_VHCI_CMD_CONTROLLER_DISABLE = 2, -+ BCE_VHCI_CMD_CONTROLLER_START = 3, -+ BCE_VHCI_CMD_CONTROLLER_PAUSE = 4, -+ -+ BCE_VHCI_CMD_PORT_POWER_ON = 0x10, -+ BCE_VHCI_CMD_PORT_POWER_OFF = 0x11, -+ BCE_VHCI_CMD_PORT_RESUME = 0x12, -+ BCE_VHCI_CMD_PORT_SUSPEND = 0x13, -+ BCE_VHCI_CMD_PORT_RESET = 0x14, -+ BCE_VHCI_CMD_PORT_DISABLE = 0x15, -+ BCE_VHCI_CMD_PORT_STATUS = 0x16, -+ -+ BCE_VHCI_CMD_DEVICE_CREATE = 0x30, -+ BCE_VHCI_CMD_DEVICE_DESTROY = 0x31, -+ -+ BCE_VHCI_CMD_ENDPOINT_CREATE = 0x40, -+ BCE_VHCI_CMD_ENDPOINT_DESTROY = 0x41, -+ BCE_VHCI_CMD_ENDPOINT_SET_STATE = 0x42, -+ BCE_VHCI_CMD_ENDPOINT_RESET = 0x44, -+ -+ /* Device to host only */ -+ BCE_VHCI_CMD_ENDPOINT_REQUEST_STATE = 0x43, -+ BCE_VHCI_CMD_TRANSFER_REQUEST = 0x1000, -+ BCE_VHCI_CMD_CONTROL_TRANSFER_STATUS = 0x1005 -+}; -+ -+enum bce_vhci_endpoint_state { -+ BCE_VHCI_ENDPOINT_ACTIVE = 0, -+ BCE_VHCI_ENDPOINT_PAUSED = 1, -+ BCE_VHCI_ENDPOINT_STALLED = 2 -+}; -+ -+static inline int bce_vhci_cmd_controller_enable(struct bce_vhci_command_queue *q, u8 busNum, u16 *portMask) -+{ -+ int status; -+ struct bce_vhci_message cmd, res; -+ cmd.cmd = BCE_VHCI_CMD_CONTROLLER_ENABLE; -+ cmd.param1 = 0x7100u | busNum; -+ status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); -+ if (!status) -+ *portMask = (u16) res.param2; -+ return status; -+} -+static inline int bce_vhci_cmd_controller_disable(struct bce_vhci_command_queue *q) -+{ -+ struct bce_vhci_message cmd, res; -+ cmd.cmd = BCE_VHCI_CMD_CONTROLLER_DISABLE; -+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); -+} -+static inline int bce_vhci_cmd_controller_start(struct bce_vhci_command_queue *q) -+{ -+ struct bce_vhci_message cmd, res; -+ cmd.cmd = BCE_VHCI_CMD_CONTROLLER_START; -+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); -+} -+static inline int bce_vhci_cmd_controller_pause(struct bce_vhci_command_queue *q) -+{ -+ struct bce_vhci_message cmd, res; -+ cmd.cmd = BCE_VHCI_CMD_CONTROLLER_PAUSE; -+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); -+} -+ -+static inline int bce_vhci_cmd_port_power_on(struct bce_vhci_command_queue *q, bce_vhci_port_t port) -+{ -+ struct bce_vhci_message cmd, res; -+ cmd.cmd = BCE_VHCI_CMD_PORT_POWER_ON; -+ cmd.param1 = port; -+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); -+} -+static inline int bce_vhci_cmd_port_power_off(struct bce_vhci_command_queue *q, bce_vhci_port_t port) -+{ -+ struct bce_vhci_message cmd, res; -+ cmd.cmd = BCE_VHCI_CMD_PORT_POWER_OFF; -+ cmd.param1 = port; -+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); -+} -+static inline int bce_vhci_cmd_port_resume(struct bce_vhci_command_queue *q, bce_vhci_port_t port) -+{ -+ struct bce_vhci_message cmd, res; -+ cmd.cmd = BCE_VHCI_CMD_PORT_RESUME; -+ cmd.param1 = port; -+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); -+} -+static inline int bce_vhci_cmd_port_suspend(struct bce_vhci_command_queue *q, bce_vhci_port_t port) -+{ -+ struct bce_vhci_message cmd, res; -+ cmd.cmd = BCE_VHCI_CMD_PORT_SUSPEND; -+ cmd.param1 = port; -+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); -+} -+static inline int bce_vhci_cmd_port_reset(struct bce_vhci_command_queue *q, bce_vhci_port_t port, u32 timeout) -+{ -+ struct bce_vhci_message cmd, res; -+ cmd.cmd = BCE_VHCI_CMD_PORT_RESET; -+ cmd.param1 = port; -+ cmd.param2 = timeout; -+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); -+} -+static inline int bce_vhci_cmd_port_disable(struct bce_vhci_command_queue *q, bce_vhci_port_t port) -+{ -+ struct bce_vhci_message cmd, res; -+ cmd.cmd = BCE_VHCI_CMD_PORT_DISABLE; -+ cmd.param1 = port; -+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); -+} -+static inline int bce_vhci_cmd_port_status(struct bce_vhci_command_queue *q, bce_vhci_port_t port, -+ u32 clearFlags, u32 *resStatus) -+{ -+ int status; -+ struct bce_vhci_message cmd, res; -+ cmd.cmd = BCE_VHCI_CMD_PORT_STATUS; -+ cmd.param1 = port; -+ cmd.param2 = clearFlags & 0x560000; -+ status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); -+ if (status >= 0) -+ *resStatus = (u32) res.param2; -+ return status; -+} -+ -+static inline int bce_vhci_cmd_device_create(struct bce_vhci_command_queue *q, bce_vhci_port_t port, -+ bce_vhci_device_t *dev) -+{ -+ int status; -+ struct bce_vhci_message cmd, res; -+ cmd.cmd = BCE_VHCI_CMD_DEVICE_CREATE; -+ cmd.param1 = port; -+ status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); -+ if (!status) -+ *dev = (bce_vhci_device_t) res.param2; -+ return status; -+} -+static inline int bce_vhci_cmd_device_destroy(struct bce_vhci_command_queue *q, bce_vhci_device_t dev) -+{ -+ struct bce_vhci_message cmd, res; -+ cmd.cmd = BCE_VHCI_CMD_DEVICE_DESTROY; -+ cmd.param1 = dev; -+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); -+} -+ -+static inline int bce_vhci_cmd_endpoint_create(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, -+ struct usb_endpoint_descriptor *desc) -+{ -+ struct bce_vhci_message cmd, res; -+ int endpoint_type = usb_endpoint_type(desc); -+ int maxp = usb_endpoint_maxp(desc); -+ int maxp_burst = usb_endpoint_maxp_mult(desc) * maxp; -+ u8 max_active_requests_pow2 = 0; -+ cmd.cmd = BCE_VHCI_CMD_ENDPOINT_CREATE; -+ cmd.param1 = dev | ((desc->bEndpointAddress & 0x8Fu) << 8); -+ if (endpoint_type == USB_ENDPOINT_XFER_BULK) -+ max_active_requests_pow2 = BCE_VHCI_BULK_MAX_ACTIVE_URBS_POW2; -+ cmd.param2 = endpoint_type | ((max_active_requests_pow2 & 0xf) << 4) | (maxp << 16) | ((u64) maxp_burst << 32); -+ if (endpoint_type == USB_ENDPOINT_XFER_INT) -+ cmd.param2 |= (desc->bInterval - 1) << 8; -+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); -+} -+static inline int bce_vhci_cmd_endpoint_destroy(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, u8 endpoint) -+{ -+ struct bce_vhci_message cmd, res; -+ cmd.cmd = BCE_VHCI_CMD_ENDPOINT_DESTROY; -+ cmd.param1 = dev | (endpoint << 8); -+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); -+} -+static inline int bce_vhci_cmd_endpoint_set_state(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, u8 endpoint, -+ enum bce_vhci_endpoint_state newState, enum bce_vhci_endpoint_state *retState) -+{ -+ int status; -+ struct bce_vhci_message cmd, res; -+ cmd.cmd = BCE_VHCI_CMD_ENDPOINT_SET_STATE; -+ cmd.param1 = dev | (endpoint << 8); -+ cmd.param2 = (u64) newState; -+ status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); -+ if (status != BCE_VHCI_INTERNAL_ERROR && status != BCE_VHCI_NO_POWER) -+ *retState = (enum bce_vhci_endpoint_state) res.param2; -+ return status; -+} -+static inline int bce_vhci_cmd_endpoint_reset(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, u8 endpoint) -+{ -+ struct bce_vhci_message cmd, res; -+ cmd.cmd = BCE_VHCI_CMD_ENDPOINT_RESET; -+ cmd.param1 = dev | (endpoint << 8); -+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); -+} -+ -+ -+#endif //BCE_VHCI_COMMAND_H -diff --git a/drivers/staging/apple-bce/vhci/queue.c b/drivers/staging/apple-bce/vhci/queue.c -new file mode 100644 -index 000000000000..7b0b5027157b ---- /dev/null -+++ b/drivers/staging/apple-bce/vhci/queue.c -@@ -0,0 +1,268 @@ -+#include "queue.h" -+#include "vhci.h" -+#include "../apple_bce.h" -+ -+ -+static void bce_vhci_message_queue_completion(struct bce_queue_sq *sq); -+ -+int bce_vhci_message_queue_create(struct bce_vhci *vhci, struct bce_vhci_message_queue *ret, const char *name) -+{ -+ int status; -+ ret->cq = bce_create_cq(vhci->dev, VHCI_EVENT_QUEUE_EL_COUNT); -+ if (!ret->cq) -+ return -EINVAL; -+ ret->sq = bce_create_sq(vhci->dev, ret->cq, name, VHCI_EVENT_QUEUE_EL_COUNT, DMA_TO_DEVICE, -+ bce_vhci_message_queue_completion, ret); -+ if (!ret->sq) { -+ status = -EINVAL; -+ goto fail_cq; -+ } -+ ret->data = dma_alloc_coherent(&vhci->dev->pci->dev, sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, -+ &ret->dma_addr, GFP_KERNEL); -+ if (!ret->data) { -+ status = -EINVAL; -+ goto fail_sq; -+ } -+ return 0; -+ -+fail_sq: -+ bce_destroy_sq(vhci->dev, ret->sq); -+ ret->sq = NULL; -+fail_cq: -+ bce_destroy_cq(vhci->dev, ret->cq); -+ ret->cq = NULL; -+ return status; -+} -+ -+void bce_vhci_message_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_message_queue *q) -+{ -+ if (!q->cq) -+ return; -+ dma_free_coherent(&vhci->dev->pci->dev, sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, -+ q->data, q->dma_addr); -+ bce_destroy_sq(vhci->dev, q->sq); -+ bce_destroy_cq(vhci->dev, q->cq); -+} -+ -+void bce_vhci_message_queue_write(struct bce_vhci_message_queue *q, struct bce_vhci_message *req) -+{ -+ int sidx; -+ struct bce_qe_submission *s; -+ sidx = q->sq->tail; -+ s = bce_next_submission(q->sq); -+ pr_debug("bce-vhci: Send message: %x s=%x p1=%x p2=%llx\n", req->cmd, req->status, req->param1, req->param2); -+ q->data[sidx] = *req; -+ bce_set_submission_single(s, q->dma_addr + sizeof(struct bce_vhci_message) * sidx, -+ sizeof(struct bce_vhci_message)); -+ bce_submit_to_device(q->sq); -+} -+ -+static void bce_vhci_message_queue_completion(struct bce_queue_sq *sq) -+{ -+ while (bce_next_completion(sq)) -+ bce_notify_submission_complete(sq); -+} -+ -+ -+ -+static void bce_vhci_event_queue_completion(struct bce_queue_sq *sq); -+ -+int __bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, -+ bce_sq_completion compl) -+{ -+ ret->vhci = vhci; -+ -+ ret->sq = bce_create_sq(vhci->dev, vhci->ev_cq, name, VHCI_EVENT_QUEUE_EL_COUNT, DMA_FROM_DEVICE, compl, ret); -+ if (!ret->sq) -+ return -EINVAL; -+ ret->data = dma_alloc_coherent(&vhci->dev->pci->dev, sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, -+ &ret->dma_addr, GFP_KERNEL); -+ if (!ret->data) { -+ bce_destroy_sq(vhci->dev, ret->sq); -+ ret->sq = NULL; -+ return -EINVAL; -+ } -+ -+ init_completion(&ret->queue_empty_completion); -+ bce_vhci_event_queue_submit_pending(ret, VHCI_EVENT_PENDING_COUNT); -+ return 0; -+} -+ -+int bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, -+ bce_vhci_event_queue_callback cb) -+{ -+ ret->cb = cb; -+ return __bce_vhci_event_queue_create(vhci, ret, name, bce_vhci_event_queue_completion); -+} -+ -+void bce_vhci_event_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_event_queue *q) -+{ -+ if (!q->sq) -+ return; -+ dma_free_coherent(&vhci->dev->pci->dev, sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, -+ q->data, q->dma_addr); -+ bce_destroy_sq(vhci->dev, q->sq); -+} -+ -+static void bce_vhci_event_queue_completion(struct bce_queue_sq *sq) -+{ -+ struct bce_sq_completion_data *cd; -+ struct bce_vhci_event_queue *ev = sq->userdata; -+ struct bce_vhci_message *msg; -+ size_t cnt = 0; -+ -+ while ((cd = bce_next_completion(sq))) { -+ if (cd->status == BCE_COMPLETION_ABORTED) { /* We flushed the queue */ -+ bce_notify_submission_complete(sq); -+ continue; -+ } -+ msg = &ev->data[sq->head]; -+ pr_debug("bce-vhci: Got event: %x s=%x p1=%x p2=%llx\n", msg->cmd, msg->status, msg->param1, msg->param2); -+ ev->cb(ev, msg); -+ -+ bce_notify_submission_complete(sq); -+ ++cnt; -+ } -+ bce_vhci_event_queue_submit_pending(ev, cnt); -+ if (atomic_read(&sq->available_commands) == sq->el_count - 1) -+ complete(&ev->queue_empty_completion); -+} -+ -+void bce_vhci_event_queue_submit_pending(struct bce_vhci_event_queue *q, size_t count) -+{ -+ int idx; -+ struct bce_qe_submission *s; -+ while (count--) { -+ if (bce_reserve_submission(q->sq, NULL)) { -+ pr_err("bce-vhci: Failed to reserve an event queue submission\n"); -+ break; -+ } -+ idx = q->sq->tail; -+ s = bce_next_submission(q->sq); -+ bce_set_submission_single(s, -+ q->dma_addr + idx * sizeof(struct bce_vhci_message), sizeof(struct bce_vhci_message)); -+ } -+ bce_submit_to_device(q->sq); -+} -+ -+void bce_vhci_event_queue_pause(struct bce_vhci_event_queue *q) -+{ -+ unsigned long timeout; -+ reinit_completion(&q->queue_empty_completion); -+ if (bce_cmd_flush_memory_queue(q->vhci->dev->cmd_cmdq, q->sq->qid)) -+ pr_warn("bce-vhci: failed to flush event queue\n"); -+ timeout = msecs_to_jiffies(5000); -+ while (atomic_read(&q->sq->available_commands) != q->sq->el_count - 1) { -+ timeout = wait_for_completion_timeout(&q->queue_empty_completion, timeout); -+ if (timeout == 0) { -+ pr_err("bce-vhci: waiting for queue to be flushed timed out\n"); -+ break; -+ } -+ } -+} -+ -+void bce_vhci_event_queue_resume(struct bce_vhci_event_queue *q) -+{ -+ if (atomic_read(&q->sq->available_commands) != q->sq->el_count - 1) { -+ pr_err("bce-vhci: resume of a queue with pending submissions\n"); -+ return; -+ } -+ bce_vhci_event_queue_submit_pending(q, VHCI_EVENT_PENDING_COUNT); -+} -+ -+void bce_vhci_command_queue_create(struct bce_vhci_command_queue *ret, struct bce_vhci_message_queue *mq) -+{ -+ ret->mq = mq; -+ ret->completion.result = NULL; -+ init_completion(&ret->completion.completion); -+ spin_lock_init(&ret->completion_lock); -+ mutex_init(&ret->mutex); -+} -+ -+void bce_vhci_command_queue_destroy(struct bce_vhci_command_queue *cq) -+{ -+ spin_lock(&cq->completion_lock); -+ if (cq->completion.result) { -+ memset(cq->completion.result, 0, sizeof(struct bce_vhci_message)); -+ cq->completion.result->status = BCE_VHCI_ABORT; -+ complete(&cq->completion.completion); -+ cq->completion.result = NULL; -+ } -+ spin_unlock(&cq->completion_lock); -+ mutex_lock(&cq->mutex); -+ mutex_unlock(&cq->mutex); -+ mutex_destroy(&cq->mutex); -+} -+ -+void bce_vhci_command_queue_deliver_completion(struct bce_vhci_command_queue *cq, struct bce_vhci_message *msg) -+{ -+ struct bce_vhci_command_queue_completion *c = &cq->completion; -+ -+ spin_lock(&cq->completion_lock); -+ if (c->result) { -+ *c->result = *msg; -+ complete(&c->completion); -+ c->result = NULL; -+ } -+ spin_unlock(&cq->completion_lock); -+} -+ -+static int __bce_vhci_command_queue_execute(struct bce_vhci_command_queue *cq, struct bce_vhci_message *req, -+ struct bce_vhci_message *res, unsigned long timeout) -+{ -+ int status; -+ struct bce_vhci_command_queue_completion *c; -+ struct bce_vhci_message creq; -+ c = &cq->completion; -+ -+ if ((status = bce_reserve_submission(cq->mq->sq, &timeout))) -+ return status; -+ -+ spin_lock(&cq->completion_lock); -+ c->result = res; -+ reinit_completion(&c->completion); -+ spin_unlock(&cq->completion_lock); -+ -+ bce_vhci_message_queue_write(cq->mq, req); -+ -+ if (!wait_for_completion_timeout(&c->completion, timeout)) { -+ /* we ran out of time, send cancellation */ -+ pr_debug("bce-vhci: command timed out req=%x\n", req->cmd); -+ if ((status = bce_reserve_submission(cq->mq->sq, &timeout))) -+ return status; -+ -+ creq = *req; -+ creq.cmd |= 0x4000; -+ bce_vhci_message_queue_write(cq->mq, &creq); -+ -+ if (!wait_for_completion_timeout(&c->completion, 1000)) { -+ pr_err("bce-vhci: Possible desync, cmd cancel timed out\n"); -+ -+ spin_lock(&cq->completion_lock); -+ c->result = NULL; -+ spin_unlock(&cq->completion_lock); -+ return -ETIMEDOUT; -+ } -+ if ((res->cmd & ~0x8000) == creq.cmd) -+ return -ETIMEDOUT; -+ /* reply for the previous command most likely arrived */ -+ } -+ -+ if ((res->cmd & ~0x8000) != req->cmd) { -+ pr_err("bce-vhci: Possible desync, cmd reply mismatch req=%x, res=%x\n", req->cmd, res->cmd); -+ return -EIO; -+ } -+ if (res->status == BCE_VHCI_SUCCESS) -+ return 0; -+ return res->status; -+} -+ -+int bce_vhci_command_queue_execute(struct bce_vhci_command_queue *cq, struct bce_vhci_message *req, -+ struct bce_vhci_message *res, unsigned long timeout) -+{ -+ int status; -+ mutex_lock(&cq->mutex); -+ status = __bce_vhci_command_queue_execute(cq, req, res, timeout); -+ mutex_unlock(&cq->mutex); -+ return status; -+} -diff --git a/drivers/staging/apple-bce/vhci/queue.h b/drivers/staging/apple-bce/vhci/queue.h -new file mode 100644 -index 000000000000..adb705b6ba1d ---- /dev/null -+++ b/drivers/staging/apple-bce/vhci/queue.h -@@ -0,0 +1,76 @@ -+#ifndef BCE_VHCI_QUEUE_H -+#define BCE_VHCI_QUEUE_H -+ -+#include -+#include "../queue.h" -+ -+#define VHCI_EVENT_QUEUE_EL_COUNT 256 -+#define VHCI_EVENT_PENDING_COUNT 32 -+ -+struct bce_vhci; -+struct bce_vhci_event_queue; -+ -+enum bce_vhci_message_status { -+ BCE_VHCI_SUCCESS = 1, -+ BCE_VHCI_ERROR = 2, -+ BCE_VHCI_USB_PIPE_STALL = 3, -+ BCE_VHCI_ABORT = 4, -+ BCE_VHCI_BAD_ARGUMENT = 5, -+ BCE_VHCI_OVERRUN = 6, -+ BCE_VHCI_INTERNAL_ERROR = 7, -+ BCE_VHCI_NO_POWER = 8, -+ BCE_VHCI_UNSUPPORTED = 9 -+}; -+struct bce_vhci_message { -+ u16 cmd; -+ u16 status; // bce_vhci_message_status -+ u32 param1; -+ u64 param2; -+}; -+ -+struct bce_vhci_message_queue { -+ struct bce_queue_cq *cq; -+ struct bce_queue_sq *sq; -+ struct bce_vhci_message *data; -+ dma_addr_t dma_addr; -+}; -+typedef void (*bce_vhci_event_queue_callback)(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg); -+struct bce_vhci_event_queue { -+ struct bce_vhci *vhci; -+ struct bce_queue_sq *sq; -+ struct bce_vhci_message *data; -+ dma_addr_t dma_addr; -+ bce_vhci_event_queue_callback cb; -+ struct completion queue_empty_completion; -+}; -+struct bce_vhci_command_queue_completion { -+ struct bce_vhci_message *result; -+ struct completion completion; -+}; -+struct bce_vhci_command_queue { -+ struct bce_vhci_message_queue *mq; -+ struct bce_vhci_command_queue_completion completion; -+ struct spinlock completion_lock; -+ struct mutex mutex; -+}; -+ -+int bce_vhci_message_queue_create(struct bce_vhci *vhci, struct bce_vhci_message_queue *ret, const char *name); -+void bce_vhci_message_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_message_queue *q); -+void bce_vhci_message_queue_write(struct bce_vhci_message_queue *q, struct bce_vhci_message *req); -+ -+int __bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, -+ bce_sq_completion compl); -+int bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, -+ bce_vhci_event_queue_callback cb); -+void bce_vhci_event_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_event_queue *q); -+void bce_vhci_event_queue_submit_pending(struct bce_vhci_event_queue *q, size_t count); -+void bce_vhci_event_queue_pause(struct bce_vhci_event_queue *q); -+void bce_vhci_event_queue_resume(struct bce_vhci_event_queue *q); -+ -+void bce_vhci_command_queue_create(struct bce_vhci_command_queue *ret, struct bce_vhci_message_queue *mq); -+void bce_vhci_command_queue_destroy(struct bce_vhci_command_queue *cq); -+int bce_vhci_command_queue_execute(struct bce_vhci_command_queue *cq, struct bce_vhci_message *req, -+ struct bce_vhci_message *res, unsigned long timeout); -+void bce_vhci_command_queue_deliver_completion(struct bce_vhci_command_queue *cq, struct bce_vhci_message *msg); -+ -+#endif //BCE_VHCI_QUEUE_H -diff --git a/drivers/staging/apple-bce/vhci/transfer.c b/drivers/staging/apple-bce/vhci/transfer.c -new file mode 100644 -index 000000000000..8226363d69c8 ---- /dev/null -+++ b/drivers/staging/apple-bce/vhci/transfer.c -@@ -0,0 +1,661 @@ -+#include "transfer.h" -+#include "../queue.h" -+#include "vhci.h" -+#include "../apple_bce.h" -+#include -+ -+static void bce_vhci_transfer_queue_completion(struct bce_queue_sq *sq); -+static void bce_vhci_transfer_queue_giveback(struct bce_vhci_transfer_queue *q); -+static void bce_vhci_transfer_queue_remove_pending(struct bce_vhci_transfer_queue *q); -+ -+static int bce_vhci_urb_init(struct bce_vhci_urb *vurb); -+static int bce_vhci_urb_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg); -+static int bce_vhci_urb_transfer_completion(struct bce_vhci_urb *urb, struct bce_sq_completion_data *c); -+ -+static void bce_vhci_transfer_queue_reset_w(struct work_struct *work); -+ -+void bce_vhci_create_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q, -+ struct usb_host_endpoint *endp, bce_vhci_device_t dev_addr, enum dma_data_direction dir) -+{ -+ char name[0x21]; -+ INIT_LIST_HEAD(&q->evq); -+ INIT_LIST_HEAD(&q->giveback_urb_list); -+ spin_lock_init(&q->urb_lock); -+ mutex_init(&q->pause_lock); -+ q->vhci = vhci; -+ q->endp = endp; -+ q->dev_addr = dev_addr; -+ q->endp_addr = (u8) (endp->desc.bEndpointAddress & 0x8F); -+ q->state = BCE_VHCI_ENDPOINT_ACTIVE; -+ q->active = true; -+ q->stalled = false; -+ q->max_active_requests = 1; -+ if (usb_endpoint_type(&endp->desc) == USB_ENDPOINT_XFER_BULK) -+ q->max_active_requests = BCE_VHCI_BULK_MAX_ACTIVE_URBS; -+ q->remaining_active_requests = q->max_active_requests; -+ q->cq = bce_create_cq(vhci->dev, 0x100); -+ INIT_WORK(&q->w_reset, bce_vhci_transfer_queue_reset_w); -+ q->sq_in = NULL; -+ if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) { -+ snprintf(name, sizeof(name), "VHC1-%i-%02x", dev_addr, 0x80 | usb_endpoint_num(&endp->desc)); -+ q->sq_in = bce_create_sq(vhci->dev, q->cq, name, 0x100, DMA_FROM_DEVICE, -+ bce_vhci_transfer_queue_completion, q); -+ } -+ q->sq_out = NULL; -+ if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) { -+ snprintf(name, sizeof(name), "VHC1-%i-%02x", dev_addr, usb_endpoint_num(&endp->desc)); -+ q->sq_out = bce_create_sq(vhci->dev, q->cq, name, 0x100, DMA_TO_DEVICE, -+ bce_vhci_transfer_queue_completion, q); -+ } -+} -+ -+void bce_vhci_destroy_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q) -+{ -+ bce_vhci_transfer_queue_giveback(q); -+ bce_vhci_transfer_queue_remove_pending(q); -+ if (q->sq_in) -+ bce_destroy_sq(vhci->dev, q->sq_in); -+ if (q->sq_out) -+ bce_destroy_sq(vhci->dev, q->sq_out); -+ bce_destroy_cq(vhci->dev, q->cq); -+} -+ -+static inline bool bce_vhci_transfer_queue_can_init_urb(struct bce_vhci_transfer_queue *q) -+{ -+ return q->remaining_active_requests > 0; -+} -+ -+static void bce_vhci_transfer_queue_defer_event(struct bce_vhci_transfer_queue *q, struct bce_vhci_message *msg) -+{ -+ struct bce_vhci_list_message *lm; -+ lm = kmalloc(sizeof(struct bce_vhci_list_message), GFP_KERNEL); -+ INIT_LIST_HEAD(&lm->list); -+ lm->msg = *msg; -+ list_add_tail(&lm->list, &q->evq); -+} -+ -+static void bce_vhci_transfer_queue_giveback(struct bce_vhci_transfer_queue *q) -+{ -+ unsigned long flags; -+ struct urb *urb; -+ spin_lock_irqsave(&q->urb_lock, flags); -+ while (!list_empty(&q->giveback_urb_list)) { -+ urb = list_first_entry(&q->giveback_urb_list, struct urb, urb_list); -+ list_del(&urb->urb_list); -+ -+ spin_unlock_irqrestore(&q->urb_lock, flags); -+ usb_hcd_giveback_urb(q->vhci->hcd, urb, urb->status); -+ spin_lock_irqsave(&q->urb_lock, flags); -+ } -+ spin_unlock_irqrestore(&q->urb_lock, flags); -+} -+ -+static void bce_vhci_transfer_queue_init_pending_urbs(struct bce_vhci_transfer_queue *q); -+ -+static void bce_vhci_transfer_queue_deliver_pending(struct bce_vhci_transfer_queue *q) -+{ -+ struct urb *urb; -+ struct bce_vhci_list_message *lm; -+ -+ while (!list_empty(&q->endp->urb_list) && !list_empty(&q->evq)) { -+ urb = list_first_entry(&q->endp->urb_list, struct urb, urb_list); -+ -+ lm = list_first_entry(&q->evq, struct bce_vhci_list_message, list); -+ if (bce_vhci_urb_update(urb->hcpriv, &lm->msg) == -EAGAIN) -+ break; -+ list_del(&lm->list); -+ kfree(lm); -+ } -+ -+ /* some of the URBs could have been completed, so initialize more URBs if possible */ -+ bce_vhci_transfer_queue_init_pending_urbs(q); -+} -+ -+static void bce_vhci_transfer_queue_remove_pending(struct bce_vhci_transfer_queue *q) -+{ -+ unsigned long flags; -+ struct bce_vhci_list_message *lm; -+ spin_lock_irqsave(&q->urb_lock, flags); -+ while (!list_empty(&q->evq)) { -+ lm = list_first_entry(&q->evq, struct bce_vhci_list_message, list); -+ list_del(&lm->list); -+ kfree(lm); -+ } -+ spin_unlock_irqrestore(&q->urb_lock, flags); -+} -+ -+void bce_vhci_transfer_queue_event(struct bce_vhci_transfer_queue *q, struct bce_vhci_message *msg) -+{ -+ unsigned long flags; -+ struct bce_vhci_urb *turb; -+ struct urb *urb; -+ spin_lock_irqsave(&q->urb_lock, flags); -+ bce_vhci_transfer_queue_deliver_pending(q); -+ -+ if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST && -+ (!list_empty(&q->evq) || list_empty(&q->endp->urb_list))) { -+ bce_vhci_transfer_queue_defer_event(q, msg); -+ goto complete; -+ } -+ if (list_empty(&q->endp->urb_list)) { -+ pr_err("bce-vhci: [%02x] Unexpected transfer queue event\n", q->endp_addr); -+ goto complete; -+ } -+ urb = list_first_entry(&q->endp->urb_list, struct urb, urb_list); -+ turb = urb->hcpriv; -+ if (bce_vhci_urb_update(turb, msg) == -EAGAIN) { -+ bce_vhci_transfer_queue_defer_event(q, msg); -+ } else { -+ bce_vhci_transfer_queue_init_pending_urbs(q); -+ } -+ -+complete: -+ spin_unlock_irqrestore(&q->urb_lock, flags); -+ bce_vhci_transfer_queue_giveback(q); -+} -+ -+static void bce_vhci_transfer_queue_completion(struct bce_queue_sq *sq) -+{ -+ unsigned long flags; -+ struct bce_sq_completion_data *c; -+ struct urb *urb; -+ struct bce_vhci_transfer_queue *q = sq->userdata; -+ spin_lock_irqsave(&q->urb_lock, flags); -+ while ((c = bce_next_completion(sq))) { -+ if (c->status == BCE_COMPLETION_ABORTED) { /* We flushed the queue */ -+ pr_debug("bce-vhci: [%02x] Got an abort completion\n", q->endp_addr); -+ bce_notify_submission_complete(sq); -+ continue; -+ } -+ if (list_empty(&q->endp->urb_list)) { -+ pr_err("bce-vhci: [%02x] Got a completion while no requests are pending\n", q->endp_addr); -+ continue; -+ } -+ pr_debug("bce-vhci: [%02x] Got a transfer queue completion\n", q->endp_addr); -+ urb = list_first_entry(&q->endp->urb_list, struct urb, urb_list); -+ bce_vhci_urb_transfer_completion(urb->hcpriv, c); -+ bce_notify_submission_complete(sq); -+ } -+ bce_vhci_transfer_queue_deliver_pending(q); -+ spin_unlock_irqrestore(&q->urb_lock, flags); -+ bce_vhci_transfer_queue_giveback(q); -+} -+ -+int bce_vhci_transfer_queue_do_pause(struct bce_vhci_transfer_queue *q) -+{ -+ unsigned long flags; -+ int status; -+ u8 endp_addr = (u8) (q->endp->desc.bEndpointAddress & 0x8F); -+ spin_lock_irqsave(&q->urb_lock, flags); -+ q->active = false; -+ spin_unlock_irqrestore(&q->urb_lock, flags); -+ if (q->sq_out) { -+ pr_err("bce-vhci: Not implemented: wait for pending output requests\n"); -+ } -+ bce_vhci_transfer_queue_remove_pending(q); -+ if ((status = bce_vhci_cmd_endpoint_set_state( -+ &q->vhci->cq, q->dev_addr, endp_addr, BCE_VHCI_ENDPOINT_PAUSED, &q->state))) -+ return status; -+ if (q->state != BCE_VHCI_ENDPOINT_PAUSED) -+ return -EINVAL; -+ if (q->sq_in) -+ bce_cmd_flush_memory_queue(q->vhci->dev->cmd_cmdq, (u16) q->sq_in->qid); -+ if (q->sq_out) -+ bce_cmd_flush_memory_queue(q->vhci->dev->cmd_cmdq, (u16) q->sq_out->qid); -+ return 0; -+} -+ -+static void bce_vhci_urb_resume(struct bce_vhci_urb *urb); -+ -+int bce_vhci_transfer_queue_do_resume(struct bce_vhci_transfer_queue *q) -+{ -+ unsigned long flags; -+ int status; -+ struct urb *urb, *urbt; -+ struct bce_vhci_urb *vurb; -+ u8 endp_addr = (u8) (q->endp->desc.bEndpointAddress & 0x8F); -+ if ((status = bce_vhci_cmd_endpoint_set_state( -+ &q->vhci->cq, q->dev_addr, endp_addr, BCE_VHCI_ENDPOINT_ACTIVE, &q->state))) -+ return status; -+ if (q->state != BCE_VHCI_ENDPOINT_ACTIVE) -+ return -EINVAL; -+ spin_lock_irqsave(&q->urb_lock, flags); -+ q->active = true; -+ list_for_each_entry_safe(urb, urbt, &q->endp->urb_list, urb_list) { -+ vurb = urb->hcpriv; -+ if (vurb->state == BCE_VHCI_URB_INIT_PENDING) { -+ if (!bce_vhci_transfer_queue_can_init_urb(q)) -+ break; -+ bce_vhci_urb_init(vurb); -+ } else { -+ bce_vhci_urb_resume(vurb); -+ } -+ } -+ bce_vhci_transfer_queue_deliver_pending(q); -+ spin_unlock_irqrestore(&q->urb_lock, flags); -+ return 0; -+} -+ -+int bce_vhci_transfer_queue_pause(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src) -+{ -+ int ret = 0; -+ mutex_lock(&q->pause_lock); -+ if ((q->paused_by & src) != src) { -+ if (!q->paused_by) -+ ret = bce_vhci_transfer_queue_do_pause(q); -+ if (!ret) -+ q->paused_by |= src; -+ } -+ mutex_unlock(&q->pause_lock); -+ return ret; -+} -+ -+int bce_vhci_transfer_queue_resume(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src) -+{ -+ int ret = 0; -+ mutex_lock(&q->pause_lock); -+ if (q->paused_by & src) { -+ if (!(q->paused_by & ~src)) -+ ret = bce_vhci_transfer_queue_do_resume(q); -+ if (!ret) -+ q->paused_by &= ~src; -+ } -+ mutex_unlock(&q->pause_lock); -+ return ret; -+} -+ -+static void bce_vhci_transfer_queue_reset_w(struct work_struct *work) -+{ -+ unsigned long flags; -+ struct bce_vhci_transfer_queue *q = container_of(work, struct bce_vhci_transfer_queue, w_reset); -+ -+ mutex_lock(&q->pause_lock); -+ spin_lock_irqsave(&q->urb_lock, flags); -+ if (!q->stalled) { -+ spin_unlock_irqrestore(&q->urb_lock, flags); -+ mutex_unlock(&q->pause_lock); -+ return; -+ } -+ q->active = false; -+ spin_unlock_irqrestore(&q->urb_lock, flags); -+ q->paused_by |= BCE_VHCI_PAUSE_INTERNAL_WQ; -+ bce_vhci_transfer_queue_remove_pending(q); -+ if (q->sq_in) -+ bce_cmd_flush_memory_queue(q->vhci->dev->cmd_cmdq, (u16) q->sq_in->qid); -+ if (q->sq_out) -+ bce_cmd_flush_memory_queue(q->vhci->dev->cmd_cmdq, (u16) q->sq_out->qid); -+ bce_vhci_cmd_endpoint_reset(&q->vhci->cq, q->dev_addr, (u8) (q->endp->desc.bEndpointAddress & 0x8F)); -+ spin_lock_irqsave(&q->urb_lock, flags); -+ q->stalled = false; -+ spin_unlock_irqrestore(&q->urb_lock, flags); -+ mutex_unlock(&q->pause_lock); -+ bce_vhci_transfer_queue_resume(q, BCE_VHCI_PAUSE_INTERNAL_WQ); -+} -+ -+void bce_vhci_transfer_queue_request_reset(struct bce_vhci_transfer_queue *q) -+{ -+ queue_work(q->vhci->tq_state_wq, &q->w_reset); -+} -+ -+static void bce_vhci_transfer_queue_init_pending_urbs(struct bce_vhci_transfer_queue *q) -+{ -+ struct urb *urb, *urbt; -+ struct bce_vhci_urb *vurb; -+ list_for_each_entry_safe(urb, urbt, &q->endp->urb_list, urb_list) { -+ vurb = urb->hcpriv; -+ if (!bce_vhci_transfer_queue_can_init_urb(q)) -+ break; -+ if (vurb->state == BCE_VHCI_URB_INIT_PENDING) -+ bce_vhci_urb_init(vurb); -+ } -+} -+ -+ -+ -+static int bce_vhci_urb_data_start(struct bce_vhci_urb *urb, unsigned long *timeout); -+ -+int bce_vhci_urb_create(struct bce_vhci_transfer_queue *q, struct urb *urb) -+{ -+ unsigned long flags; -+ int status = 0; -+ struct bce_vhci_urb *vurb; -+ vurb = kzalloc(sizeof(struct bce_vhci_urb), GFP_KERNEL); -+ urb->hcpriv = vurb; -+ -+ vurb->q = q; -+ vurb->urb = urb; -+ vurb->dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE; -+ vurb->is_control = (usb_endpoint_num(&urb->ep->desc) == 0); -+ -+ spin_lock_irqsave(&q->urb_lock, flags); -+ status = usb_hcd_link_urb_to_ep(q->vhci->hcd, urb); -+ if (status) { -+ spin_unlock_irqrestore(&q->urb_lock, flags); -+ urb->hcpriv = NULL; -+ kfree(vurb); -+ return status; -+ } -+ -+ if (q->active) { -+ if (bce_vhci_transfer_queue_can_init_urb(vurb->q)) -+ status = bce_vhci_urb_init(vurb); -+ else -+ vurb->state = BCE_VHCI_URB_INIT_PENDING; -+ } else { -+ if (q->stalled) -+ bce_vhci_transfer_queue_request_reset(q); -+ vurb->state = BCE_VHCI_URB_INIT_PENDING; -+ } -+ if (status) { -+ usb_hcd_unlink_urb_from_ep(q->vhci->hcd, urb); -+ urb->hcpriv = NULL; -+ kfree(vurb); -+ } else { -+ bce_vhci_transfer_queue_deliver_pending(q); -+ } -+ spin_unlock_irqrestore(&q->urb_lock, flags); -+ pr_debug("bce-vhci: [%02x] URB enqueued (dir = %s, size = %i)\n", q->endp_addr, -+ usb_urb_dir_in(urb) ? "IN" : "OUT", urb->transfer_buffer_length); -+ return status; -+} -+ -+static int bce_vhci_urb_init(struct bce_vhci_urb *vurb) -+{ -+ int status = 0; -+ -+ if (vurb->q->remaining_active_requests == 0) { -+ pr_err("bce-vhci: cannot init request (remaining_active_requests = 0)\n"); -+ return -EINVAL; -+ } -+ -+ if (vurb->is_control) { -+ vurb->state = BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST; -+ } else { -+ status = bce_vhci_urb_data_start(vurb, NULL); -+ } -+ -+ if (!status) { -+ --vurb->q->remaining_active_requests; -+ } -+ return status; -+} -+ -+static void bce_vhci_urb_complete(struct bce_vhci_urb *urb, int status) -+{ -+ struct bce_vhci_transfer_queue *q = urb->q; -+ struct bce_vhci *vhci = q->vhci; -+ struct urb *real_urb = urb->urb; -+ pr_debug("bce-vhci: [%02x] URB complete %i\n", q->endp_addr, status); -+ usb_hcd_unlink_urb_from_ep(vhci->hcd, real_urb); -+ real_urb->hcpriv = NULL; -+ real_urb->status = status; -+ if (urb->state != BCE_VHCI_URB_INIT_PENDING) -+ ++urb->q->remaining_active_requests; -+ kfree(urb); -+ list_add_tail(&real_urb->urb_list, &q->giveback_urb_list); -+} -+ -+int bce_vhci_urb_request_cancel(struct bce_vhci_transfer_queue *q, struct urb *urb, int status) -+{ -+ struct bce_vhci_urb *vurb; -+ unsigned long flags; -+ int ret; -+ -+ spin_lock_irqsave(&q->urb_lock, flags); -+ if ((ret = usb_hcd_check_unlink_urb(q->vhci->hcd, urb, status))) { -+ spin_unlock_irqrestore(&q->urb_lock, flags); -+ return ret; -+ } -+ -+ vurb = urb->hcpriv; -+ /* If the URB wasn't posted to the device yet, we can still remove it on the host without pausing the queue. */ -+ if (vurb->state != BCE_VHCI_URB_INIT_PENDING) { -+ pr_debug("bce-vhci: [%02x] Cancelling URB\n", q->endp_addr); -+ -+ spin_unlock_irqrestore(&q->urb_lock, flags); -+ bce_vhci_transfer_queue_pause(q, BCE_VHCI_PAUSE_INTERNAL_WQ); -+ spin_lock_irqsave(&q->urb_lock, flags); -+ -+ ++q->remaining_active_requests; -+ } -+ -+ usb_hcd_unlink_urb_from_ep(q->vhci->hcd, urb); -+ -+ spin_unlock_irqrestore(&q->urb_lock, flags); -+ -+ usb_hcd_giveback_urb(q->vhci->hcd, urb, status); -+ -+ if (vurb->state != BCE_VHCI_URB_INIT_PENDING) -+ bce_vhci_transfer_queue_resume(q, BCE_VHCI_PAUSE_INTERNAL_WQ); -+ -+ kfree(vurb); -+ -+ return 0; -+} -+ -+static int bce_vhci_urb_data_transfer_in(struct bce_vhci_urb *urb, unsigned long *timeout) -+{ -+ struct bce_vhci_message msg; -+ struct bce_qe_submission *s; -+ u32 tr_len; -+ int reservation1, reservation2 = -EFAULT; -+ -+ pr_debug("bce-vhci: [%02x] DMA from device %llx %x\n", urb->q->endp_addr, -+ (u64) urb->urb->transfer_dma, urb->urb->transfer_buffer_length); -+ -+ /* Reserve both a message and a submission, so we don't run into issues later. */ -+ reservation1 = bce_reserve_submission(urb->q->vhci->msg_asynchronous.sq, timeout); -+ if (!reservation1) -+ reservation2 = bce_reserve_submission(urb->q->sq_in, timeout); -+ if (reservation1 || reservation2) { -+ pr_err("bce-vhci: Failed to reserve a submission for URB data transfer\n"); -+ if (!reservation1) -+ bce_cancel_submission_reservation(urb->q->vhci->msg_asynchronous.sq); -+ return -ENOMEM; -+ } -+ -+ urb->send_offset = urb->receive_offset; -+ -+ tr_len = urb->urb->transfer_buffer_length - urb->send_offset; -+ -+ spin_lock(&urb->q->vhci->msg_asynchronous_lock); -+ msg.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST; -+ msg.status = 0; -+ msg.param1 = ((urb->urb->ep->desc.bEndpointAddress & 0x8Fu) << 8) | urb->q->dev_addr; -+ msg.param2 = tr_len; -+ bce_vhci_message_queue_write(&urb->q->vhci->msg_asynchronous, &msg); -+ spin_unlock(&urb->q->vhci->msg_asynchronous_lock); -+ -+ s = bce_next_submission(urb->q->sq_in); -+ bce_set_submission_single(s, urb->urb->transfer_dma + urb->send_offset, tr_len); -+ bce_submit_to_device(urb->q->sq_in); -+ -+ urb->state = BCE_VHCI_URB_WAITING_FOR_COMPLETION; -+ return 0; -+} -+ -+static int bce_vhci_urb_data_start(struct bce_vhci_urb *urb, unsigned long *timeout) -+{ -+ if (urb->dir == DMA_TO_DEVICE) { -+ if (urb->urb->transfer_buffer_length > 0) -+ urb->state = BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST; -+ else -+ urb->state = BCE_VHCI_URB_DATA_TRANSFER_COMPLETE; -+ return 0; -+ } else { -+ return bce_vhci_urb_data_transfer_in(urb, timeout); -+ } -+} -+ -+static int bce_vhci_urb_send_out_data(struct bce_vhci_urb *urb, dma_addr_t addr, size_t size) -+{ -+ struct bce_qe_submission *s; -+ unsigned long timeout = 0; -+ if (bce_reserve_submission(urb->q->sq_out, &timeout)) { -+ pr_err("bce-vhci: Failed to reserve a submission for URB data transfer\n"); -+ return -EPIPE; -+ } -+ -+ pr_debug("bce-vhci: [%02x] DMA to device %llx %lx\n", urb->q->endp_addr, (u64) addr, size); -+ -+ s = bce_next_submission(urb->q->sq_out); -+ bce_set_submission_single(s, addr, size); -+ bce_submit_to_device(urb->q->sq_out); -+ return 0; -+} -+ -+static int bce_vhci_urb_data_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg) -+{ -+ u32 tr_len; -+ int status; -+ if (urb->state == BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST) { -+ if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST) { -+ tr_len = min(urb->urb->transfer_buffer_length - urb->send_offset, (u32) msg->param2); -+ if ((status = bce_vhci_urb_send_out_data(urb, urb->urb->transfer_dma + urb->send_offset, tr_len))) -+ return status; -+ urb->send_offset += tr_len; -+ urb->state = BCE_VHCI_URB_WAITING_FOR_COMPLETION; -+ return 0; -+ } -+ } -+ -+ /* 0x1000 in out queues aren't really unexpected */ -+ if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST && urb->q->sq_out != NULL) -+ return -EAGAIN; -+ pr_err("bce-vhci: [%02x] %s URB unexpected message (state = %x, msg: %x %x %x %llx)\n", -+ urb->q->endp_addr, (urb->is_control ? "Control (data update)" : "Data"), urb->state, -+ msg->cmd, msg->status, msg->param1, msg->param2); -+ return -EAGAIN; -+} -+ -+static int bce_vhci_urb_data_transfer_completion(struct bce_vhci_urb *urb, struct bce_sq_completion_data *c) -+{ -+ if (urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { -+ urb->receive_offset += c->data_size; -+ if (urb->dir == DMA_FROM_DEVICE || urb->receive_offset >= urb->urb->transfer_buffer_length) { -+ urb->urb->actual_length = (u32) urb->receive_offset; -+ urb->state = BCE_VHCI_URB_DATA_TRANSFER_COMPLETE; -+ if (!urb->is_control) { -+ bce_vhci_urb_complete(urb, 0); -+ return -ENOENT; -+ } -+ } -+ } else { -+ pr_err("bce-vhci: [%02x] Data URB unexpected completion\n", urb->q->endp_addr); -+ } -+ return 0; -+} -+ -+ -+static int bce_vhci_urb_control_check_status(struct bce_vhci_urb *urb) -+{ -+ struct bce_vhci_transfer_queue *q = urb->q; -+ if (urb->received_status == 0) -+ return 0; -+ if (urb->state == BCE_VHCI_URB_DATA_TRANSFER_COMPLETE || -+ (urb->received_status != BCE_VHCI_SUCCESS && urb->state != BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST && -+ urb->state != BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION)) { -+ urb->state = BCE_VHCI_URB_CONTROL_COMPLETE; -+ if (urb->received_status != BCE_VHCI_SUCCESS) { -+ pr_err("bce-vhci: [%02x] URB failed: %x\n", urb->q->endp_addr, urb->received_status); -+ urb->q->active = false; -+ urb->q->stalled = true; -+ bce_vhci_urb_complete(urb, -EPIPE); -+ if (!list_empty(&q->endp->urb_list)) -+ bce_vhci_transfer_queue_request_reset(q); -+ return -ENOENT; -+ } -+ bce_vhci_urb_complete(urb, 0); -+ return -ENOENT; -+ } -+ return 0; -+} -+ -+static int bce_vhci_urb_control_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg) -+{ -+ int status; -+ if (msg->cmd == BCE_VHCI_CMD_CONTROL_TRANSFER_STATUS) { -+ urb->received_status = msg->status; -+ return bce_vhci_urb_control_check_status(urb); -+ } -+ -+ if (urb->state == BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST) { -+ if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST) { -+ if (bce_vhci_urb_send_out_data(urb, urb->urb->setup_dma, sizeof(struct usb_ctrlrequest))) { -+ pr_err("bce-vhci: [%02x] Failed to start URB setup transfer\n", urb->q->endp_addr); -+ return 0; /* TODO: fail the URB? */ -+ } -+ urb->state = BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION; -+ pr_debug("bce-vhci: [%02x] Sent setup %llx\n", urb->q->endp_addr, urb->urb->setup_dma); -+ return 0; -+ } -+ } else if (urb->state == BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST || -+ urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { -+ if ((status = bce_vhci_urb_data_update(urb, msg))) -+ return status; -+ return bce_vhci_urb_control_check_status(urb); -+ } -+ -+ /* 0x1000 in out queues aren't really unexpected */ -+ if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST && urb->q->sq_out != NULL) -+ return -EAGAIN; -+ pr_err("bce-vhci: [%02x] Control URB unexpected message (state = %x, msg: %x %x %x %llx)\n", urb->q->endp_addr, -+ urb->state, msg->cmd, msg->status, msg->param1, msg->param2); -+ return -EAGAIN; -+} -+ -+static int bce_vhci_urb_control_transfer_completion(struct bce_vhci_urb *urb, struct bce_sq_completion_data *c) -+{ -+ int status; -+ unsigned long timeout; -+ -+ if (urb->state == BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION) { -+ if (c->data_size != sizeof(struct usb_ctrlrequest)) -+ pr_err("bce-vhci: [%02x] transfer complete data size mistmatch for usb_ctrlrequest (%llx instead of %lx)\n", -+ urb->q->endp_addr, c->data_size, sizeof(struct usb_ctrlrequest)); -+ -+ timeout = 1000; -+ status = bce_vhci_urb_data_start(urb, &timeout); -+ if (status) { -+ bce_vhci_urb_complete(urb, status); -+ return -ENOENT; -+ } -+ return 0; -+ } else if (urb->state == BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST || -+ urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { -+ if ((status = bce_vhci_urb_data_transfer_completion(urb, c))) -+ return status; -+ return bce_vhci_urb_control_check_status(urb); -+ } else { -+ pr_err("bce-vhci: [%02x] Control URB unexpected completion (state = %x)\n", urb->q->endp_addr, urb->state); -+ } -+ return 0; -+} -+ -+static int bce_vhci_urb_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg) -+{ -+ if (urb->state == BCE_VHCI_URB_INIT_PENDING) -+ return -EAGAIN; -+ if (urb->is_control) -+ return bce_vhci_urb_control_update(urb, msg); -+ else -+ return bce_vhci_urb_data_update(urb, msg); -+} -+ -+static int bce_vhci_urb_transfer_completion(struct bce_vhci_urb *urb, struct bce_sq_completion_data *c) -+{ -+ if (urb->is_control) -+ return bce_vhci_urb_control_transfer_completion(urb, c); -+ else -+ return bce_vhci_urb_data_transfer_completion(urb, c); -+} -+ -+static void bce_vhci_urb_resume(struct bce_vhci_urb *urb) -+{ -+ int status = 0; -+ if (urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { -+ status = bce_vhci_urb_data_transfer_in(urb, NULL); -+ } -+ if (status) -+ bce_vhci_urb_complete(urb, status); -+} -diff --git a/drivers/staging/apple-bce/vhci/transfer.h b/drivers/staging/apple-bce/vhci/transfer.h -new file mode 100644 -index 000000000000..89ecad6bcf8f ---- /dev/null -+++ b/drivers/staging/apple-bce/vhci/transfer.h -@@ -0,0 +1,73 @@ -+#ifndef BCEDRIVER_TRANSFER_H -+#define BCEDRIVER_TRANSFER_H -+ -+#include -+#include "queue.h" -+#include "command.h" -+#include "../queue.h" -+ -+struct bce_vhci_list_message { -+ struct list_head list; -+ struct bce_vhci_message msg; -+}; -+enum bce_vhci_pause_source { -+ BCE_VHCI_PAUSE_INTERNAL_WQ = 1, -+ BCE_VHCI_PAUSE_FIRMWARE = 2, -+ BCE_VHCI_PAUSE_SUSPEND = 4, -+ BCE_VHCI_PAUSE_SHUTDOWN = 8 -+}; -+struct bce_vhci_transfer_queue { -+ struct bce_vhci *vhci; -+ struct usb_host_endpoint *endp; -+ enum bce_vhci_endpoint_state state; -+ u32 max_active_requests, remaining_active_requests; -+ bool active, stalled; -+ u32 paused_by; -+ bce_vhci_device_t dev_addr; -+ u8 endp_addr; -+ struct bce_queue_cq *cq; -+ struct bce_queue_sq *sq_in; -+ struct bce_queue_sq *sq_out; -+ struct list_head evq; -+ struct spinlock urb_lock; -+ struct mutex pause_lock; -+ struct list_head giveback_urb_list; -+ -+ struct work_struct w_reset; -+}; -+enum bce_vhci_urb_state { -+ BCE_VHCI_URB_INIT_PENDING, -+ -+ BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST, -+ BCE_VHCI_URB_WAITING_FOR_COMPLETION, -+ BCE_VHCI_URB_DATA_TRANSFER_COMPLETE, -+ -+ BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST, -+ BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION, -+ BCE_VHCI_URB_CONTROL_COMPLETE -+}; -+struct bce_vhci_urb { -+ struct urb *urb; -+ struct bce_vhci_transfer_queue *q; -+ enum dma_data_direction dir; -+ bool is_control; -+ enum bce_vhci_urb_state state; -+ int received_status; -+ u32 send_offset; -+ u32 receive_offset; -+}; -+ -+void bce_vhci_create_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q, -+ struct usb_host_endpoint *endp, bce_vhci_device_t dev_addr, enum dma_data_direction dir); -+void bce_vhci_destroy_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q); -+void bce_vhci_transfer_queue_event(struct bce_vhci_transfer_queue *q, struct bce_vhci_message *msg); -+int bce_vhci_transfer_queue_do_pause(struct bce_vhci_transfer_queue *q); -+int bce_vhci_transfer_queue_do_resume(struct bce_vhci_transfer_queue *q); -+int bce_vhci_transfer_queue_pause(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src); -+int bce_vhci_transfer_queue_resume(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src); -+void bce_vhci_transfer_queue_request_reset(struct bce_vhci_transfer_queue *q); -+ -+int bce_vhci_urb_create(struct bce_vhci_transfer_queue *q, struct urb *urb); -+int bce_vhci_urb_request_cancel(struct bce_vhci_transfer_queue *q, struct urb *urb, int status); -+ -+#endif //BCEDRIVER_TRANSFER_H -diff --git a/drivers/staging/apple-bce/vhci/vhci.c b/drivers/staging/apple-bce/vhci/vhci.c -new file mode 100644 -index 000000000000..eb26f55000d8 ---- /dev/null -+++ b/drivers/staging/apple-bce/vhci/vhci.c -@@ -0,0 +1,759 @@ -+#include "vhci.h" -+#include "../apple_bce.h" -+#include "command.h" -+#include -+#include -+#include -+#include -+ -+static dev_t bce_vhci_chrdev; -+static struct class *bce_vhci_class; -+static const struct hc_driver bce_vhci_driver; -+static u16 bce_vhci_port_mask = U16_MAX; -+ -+static int bce_vhci_create_event_queues(struct bce_vhci *vhci); -+static void bce_vhci_destroy_event_queues(struct bce_vhci *vhci); -+static int bce_vhci_create_message_queues(struct bce_vhci *vhci); -+static void bce_vhci_destroy_message_queues(struct bce_vhci *vhci); -+static void bce_vhci_handle_firmware_events_w(struct work_struct *ws); -+static void bce_vhci_firmware_event_completion(struct bce_queue_sq *sq); -+ -+int bce_vhci_create(struct apple_bce_device *dev, struct bce_vhci *vhci) -+{ -+ int status; -+ -+ spin_lock_init(&vhci->hcd_spinlock); -+ -+ vhci->dev = dev; -+ -+ vhci->vdevt = bce_vhci_chrdev; -+ vhci->vdev = device_create(bce_vhci_class, dev->dev, vhci->vdevt, NULL, "bce-vhci"); -+ if (IS_ERR_OR_NULL(vhci->vdev)) { -+ status = PTR_ERR(vhci->vdev); -+ goto fail_dev; -+ } -+ -+ if ((status = bce_vhci_create_message_queues(vhci))) -+ goto fail_mq; -+ if ((status = bce_vhci_create_event_queues(vhci))) -+ goto fail_eq; -+ -+ vhci->tq_state_wq = alloc_ordered_workqueue("bce-vhci-tq-state", 0); -+ INIT_WORK(&vhci->w_fw_events, bce_vhci_handle_firmware_events_w); -+ -+ vhci->hcd = usb_create_hcd(&bce_vhci_driver, vhci->vdev, "bce-vhci"); -+ if (!vhci->hcd) { -+ status = -ENOMEM; -+ goto fail_hcd; -+ } -+ vhci->hcd->self.sysdev = &dev->pci->dev; -+#if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0) -+ vhci->hcd->self.uses_dma = 1; -+#endif -+ *((struct bce_vhci **) vhci->hcd->hcd_priv) = vhci; -+ vhci->hcd->speed = HCD_USB2; -+ -+ if ((status = usb_add_hcd(vhci->hcd, 0, 0))) -+ goto fail_hcd; -+ -+ return 0; -+ -+fail_hcd: -+ bce_vhci_destroy_event_queues(vhci); -+fail_eq: -+ bce_vhci_destroy_message_queues(vhci); -+fail_mq: -+ device_destroy(bce_vhci_class, vhci->vdevt); -+fail_dev: -+ if (!status) -+ status = -EINVAL; -+ return status; -+} -+ -+void bce_vhci_destroy(struct bce_vhci *vhci) -+{ -+ usb_remove_hcd(vhci->hcd); -+ bce_vhci_destroy_event_queues(vhci); -+ bce_vhci_destroy_message_queues(vhci); -+ device_destroy(bce_vhci_class, vhci->vdevt); -+} -+ -+struct bce_vhci *bce_vhci_from_hcd(struct usb_hcd *hcd) -+{ -+ return *((struct bce_vhci **) hcd->hcd_priv); -+} -+ -+int bce_vhci_start(struct usb_hcd *hcd) -+{ -+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); -+ int status; -+ u16 port_mask = 0; -+ bce_vhci_port_t port_no = 0; -+ if ((status = bce_vhci_cmd_controller_enable(&vhci->cq, 1, &port_mask))) -+ return status; -+ vhci->port_mask = port_mask; -+ vhci->port_power_mask = 0; -+ if ((status = bce_vhci_cmd_controller_start(&vhci->cq))) -+ return status; -+ port_mask = vhci->port_mask; -+ while (port_mask) { -+ port_no += 1; -+ port_mask >>= 1; -+ } -+ vhci->port_count = port_no; -+ return 0; -+} -+ -+void bce_vhci_stop(struct usb_hcd *hcd) -+{ -+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); -+ bce_vhci_cmd_controller_disable(&vhci->cq); -+} -+ -+static int bce_vhci_hub_status_data(struct usb_hcd *hcd, char *buf) -+{ -+ return 0; -+} -+ -+static int bce_vhci_reset_device(struct bce_vhci *vhci, int index, u16 timeout); -+ -+static int bce_vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, u16 wIndex, char *buf, u16 wLength) -+{ -+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); -+ int status; -+ struct usb_hub_descriptor *hd; -+ struct usb_hub_status *hs; -+ struct usb_port_status *ps; -+ u32 port_status; -+ // pr_info("bce-vhci: bce_vhci_hub_control %x %i %i [bufl=%i]\n", typeReq, wValue, wIndex, wLength); -+ if (typeReq == GetHubDescriptor && wLength >= sizeof(struct usb_hub_descriptor)) { -+ hd = (struct usb_hub_descriptor *) buf; -+ memset(hd, 0, sizeof(*hd)); -+ hd->bDescLength = sizeof(struct usb_hub_descriptor); -+ hd->bDescriptorType = USB_DT_HUB; -+ hd->bNbrPorts = (u8) vhci->port_count; -+ hd->wHubCharacteristics = HUB_CHAR_INDV_PORT_LPSM | HUB_CHAR_INDV_PORT_OCPM; -+ hd->bPwrOn2PwrGood = 0; -+ hd->bHubContrCurrent = 0; -+ return 0; -+ } else if (typeReq == GetHubStatus && wLength >= sizeof(struct usb_hub_status)) { -+ hs = (struct usb_hub_status *) buf; -+ memset(hs, 0, sizeof(*hs)); -+ hs->wHubStatus = 0; -+ hs->wHubChange = 0; -+ return 0; -+ } else if (typeReq == GetPortStatus && wLength >= 4 /* usb 2.0 */) { -+ ps = (struct usb_port_status *) buf; -+ ps->wPortStatus = 0; -+ ps->wPortChange = 0; -+ -+ if (vhci->port_power_mask & BIT(wIndex)) -+ ps->wPortStatus |= USB_PORT_STAT_POWER; -+ -+ if (!(bce_vhci_port_mask & BIT(wIndex))) -+ return 0; -+ -+ if ((status = bce_vhci_cmd_port_status(&vhci->cq, (u8) wIndex, 0, &port_status))) -+ return status; -+ -+ if (port_status & 16) -+ ps->wPortStatus |= USB_PORT_STAT_ENABLE | USB_PORT_STAT_HIGH_SPEED; -+ if (port_status & 4) -+ ps->wPortStatus |= USB_PORT_STAT_CONNECTION; -+ if (port_status & 2) -+ ps->wPortStatus |= USB_PORT_STAT_OVERCURRENT; -+ if (port_status & 8) -+ ps->wPortStatus |= USB_PORT_STAT_RESET; -+ if (port_status & 0x60) -+ ps->wPortStatus |= USB_PORT_STAT_SUSPEND; -+ -+ if (port_status & 0x40000) -+ ps->wPortChange |= USB_PORT_STAT_C_CONNECTION; -+ -+ pr_debug("bce-vhci: Translated status %x to %x:%x\n", port_status, ps->wPortStatus, ps->wPortChange); -+ return 0; -+ } else if (typeReq == SetPortFeature) { -+ if (wValue == USB_PORT_FEAT_POWER) { -+ status = bce_vhci_cmd_port_power_on(&vhci->cq, (u8) wIndex); -+ /* As far as I am aware, power status is not part of the port status so store it separately */ -+ if (!status) -+ vhci->port_power_mask |= BIT(wIndex); -+ return status; -+ } -+ if (wValue == USB_PORT_FEAT_RESET) { -+ return bce_vhci_reset_device(vhci, wIndex, wValue); -+ } -+ if (wValue == USB_PORT_FEAT_SUSPEND) { -+ /* TODO: Am I supposed to also suspend the endpoints? */ -+ pr_debug("bce-vhci: Suspending port %i\n", wIndex); -+ return bce_vhci_cmd_port_suspend(&vhci->cq, (u8) wIndex); -+ } -+ } else if (typeReq == ClearPortFeature) { -+ if (wValue == USB_PORT_FEAT_ENABLE) -+ return bce_vhci_cmd_port_disable(&vhci->cq, (u8) wIndex); -+ if (wValue == USB_PORT_FEAT_POWER) { -+ status = bce_vhci_cmd_port_power_off(&vhci->cq, (u8) wIndex); -+ if (!status) -+ vhci->port_power_mask &= ~BIT(wIndex); -+ return status; -+ } -+ if (wValue == USB_PORT_FEAT_C_CONNECTION) -+ return bce_vhci_cmd_port_status(&vhci->cq, (u8) wIndex, 0x40000, &port_status); -+ if (wValue == USB_PORT_FEAT_C_RESET) { /* I don't think I can transfer it in any way */ -+ return 0; -+ } -+ if (wValue == USB_PORT_FEAT_SUSPEND) { -+ pr_debug("bce-vhci: Resuming port %i\n", wIndex); -+ return bce_vhci_cmd_port_resume(&vhci->cq, (u8) wIndex); -+ } -+ } -+ pr_err("bce-vhci: bce_vhci_hub_control unhandled request: %x %i %i [bufl=%i]\n", typeReq, wValue, wIndex, wLength); -+ dump_stack(); -+ return -EIO; -+} -+ -+static int bce_vhci_enable_device(struct usb_hcd *hcd, struct usb_device *udev) -+{ -+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); -+ struct bce_vhci_device *vdev; -+ bce_vhci_device_t devid; -+ pr_info("bce_vhci_enable_device\n"); -+ -+ if (vhci->port_to_device[udev->portnum]) -+ return 0; -+ -+ /* We need to early address the device */ -+ if (bce_vhci_cmd_device_create(&vhci->cq, udev->portnum, &devid)) -+ return -EIO; -+ -+ pr_info("bce_vhci_cmd_device_create %i -> %i\n", udev->portnum, devid); -+ -+ vdev = kzalloc(sizeof(struct bce_vhci_device), GFP_KERNEL); -+ vhci->port_to_device[udev->portnum] = devid; -+ vhci->devices[devid] = vdev; -+ -+ bce_vhci_create_transfer_queue(vhci, &vdev->tq[0], &udev->ep0, devid, DMA_BIDIRECTIONAL); -+ udev->ep0.hcpriv = &vdev->tq[0]; -+ vdev->tq_mask |= BIT(0); -+ -+ bce_vhci_cmd_endpoint_create(&vhci->cq, devid, &udev->ep0.desc); -+ return 0; -+} -+ -+static int bce_vhci_address_device(struct usb_hcd *hcd, struct usb_device *udev, unsigned int timeout_ms) //TODO: follow timeout -+{ -+ /* This is the same as enable_device, but instead in the old scheme */ -+ return bce_vhci_enable_device(hcd, udev); -+} -+ -+static void bce_vhci_free_device(struct usb_hcd *hcd, struct usb_device *udev) -+{ -+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); -+ int i; -+ bce_vhci_device_t devid; -+ struct bce_vhci_device *dev; -+ pr_info("bce_vhci_free_device %i\n", udev->portnum); -+ if (!vhci->port_to_device[udev->portnum]) -+ return; -+ devid = vhci->port_to_device[udev->portnum]; -+ dev = vhci->devices[devid]; -+ for (i = 0; i < 32; i++) { -+ if (dev->tq_mask & BIT(i)) { -+ bce_vhci_transfer_queue_pause(&dev->tq[i], BCE_VHCI_PAUSE_SHUTDOWN); -+ bce_vhci_cmd_endpoint_destroy(&vhci->cq, devid, (u8) i); -+ bce_vhci_destroy_transfer_queue(vhci, &dev->tq[i]); -+ } -+ } -+ vhci->devices[devid] = NULL; -+ vhci->port_to_device[udev->portnum] = 0; -+ bce_vhci_cmd_device_destroy(&vhci->cq, devid); -+ kfree(dev); -+} -+ -+static int bce_vhci_reset_device(struct bce_vhci *vhci, int index, u16 timeout) -+{ -+ struct bce_vhci_device *dev = NULL; -+ bce_vhci_device_t devid; -+ int i; -+ int status; -+ enum dma_data_direction dir; -+ pr_info("bce_vhci_reset_device %i\n", index); -+ -+ devid = vhci->port_to_device[index]; -+ if (devid) { -+ dev = vhci->devices[devid]; -+ -+ for (i = 0; i < 32; i++) { -+ if (dev->tq_mask & BIT(i)) { -+ bce_vhci_transfer_queue_pause(&dev->tq[i], BCE_VHCI_PAUSE_SHUTDOWN); -+ bce_vhci_cmd_endpoint_destroy(&vhci->cq, devid, (u8) i); -+ bce_vhci_destroy_transfer_queue(vhci, &dev->tq[i]); -+ } -+ } -+ vhci->devices[devid] = NULL; -+ vhci->port_to_device[index] = 0; -+ bce_vhci_cmd_device_destroy(&vhci->cq, devid); -+ } -+ status = bce_vhci_cmd_port_reset(&vhci->cq, (u8) index, timeout); -+ -+ if (dev) { -+ if ((status = bce_vhci_cmd_device_create(&vhci->cq, index, &devid))) -+ return status; -+ vhci->devices[devid] = dev; -+ vhci->port_to_device[index] = devid; -+ -+ for (i = 0; i < 32; i++) { -+ if (dev->tq_mask & BIT(i)) { -+ dir = usb_endpoint_dir_in(&dev->tq[i].endp->desc) ? DMA_FROM_DEVICE : DMA_TO_DEVICE; -+ if (i == 0) -+ dir = DMA_BIDIRECTIONAL; -+ bce_vhci_create_transfer_queue(vhci, &dev->tq[i], dev->tq[i].endp, devid, dir); -+ bce_vhci_cmd_endpoint_create(&vhci->cq, devid, &dev->tq[i].endp->desc); -+ } -+ } -+ } -+ -+ return status; -+} -+ -+static int bce_vhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev) -+{ -+ return 0; -+} -+ -+static int bce_vhci_get_frame_number(struct usb_hcd *hcd) -+{ -+ return 0; -+} -+ -+static int bce_vhci_bus_suspend(struct usb_hcd *hcd) -+{ -+ int i, j; -+ int status; -+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); -+ pr_info("bce_vhci: suspend started\n"); -+ -+ pr_info("bce_vhci: suspend endpoints\n"); -+ for (i = 0; i < 16; i++) { -+ if (!vhci->port_to_device[i]) -+ continue; -+ for (j = 0; j < 32; j++) { -+ if (!(vhci->devices[vhci->port_to_device[i]]->tq_mask & BIT(j))) -+ continue; -+ bce_vhci_transfer_queue_pause(&vhci->devices[vhci->port_to_device[i]]->tq[j], -+ BCE_VHCI_PAUSE_SUSPEND); -+ } -+ } -+ -+ pr_info("bce_vhci: suspend ports\n"); -+ for (i = 0; i < 16; i++) { -+ if (!vhci->port_to_device[i]) -+ continue; -+ bce_vhci_cmd_port_suspend(&vhci->cq, i); -+ } -+ pr_info("bce_vhci: suspend controller\n"); -+ if ((status = bce_vhci_cmd_controller_pause(&vhci->cq))) -+ return status; -+ -+ bce_vhci_event_queue_pause(&vhci->ev_commands); -+ bce_vhci_event_queue_pause(&vhci->ev_system); -+ bce_vhci_event_queue_pause(&vhci->ev_isochronous); -+ bce_vhci_event_queue_pause(&vhci->ev_interrupt); -+ bce_vhci_event_queue_pause(&vhci->ev_asynchronous); -+ pr_info("bce_vhci: suspend done\n"); -+ return 0; -+} -+ -+static int bce_vhci_bus_resume(struct usb_hcd *hcd) -+{ -+ int i, j; -+ int status; -+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); -+ pr_info("bce_vhci: resume started\n"); -+ -+ bce_vhci_event_queue_resume(&vhci->ev_system); -+ bce_vhci_event_queue_resume(&vhci->ev_isochronous); -+ bce_vhci_event_queue_resume(&vhci->ev_interrupt); -+ bce_vhci_event_queue_resume(&vhci->ev_asynchronous); -+ bce_vhci_event_queue_resume(&vhci->ev_commands); -+ -+ pr_info("bce_vhci: resume controller\n"); -+ if ((status = bce_vhci_cmd_controller_start(&vhci->cq))) -+ return status; -+ -+ pr_info("bce_vhci: resume ports\n"); -+ for (i = 0; i < 16; i++) { -+ if (!vhci->port_to_device[i]) -+ continue; -+ bce_vhci_cmd_port_resume(&vhci->cq, i); -+ } -+ pr_info("bce_vhci: resume endpoints\n"); -+ for (i = 0; i < 16; i++) { -+ if (!vhci->port_to_device[i]) -+ continue; -+ for (j = 0; j < 32; j++) { -+ if (!(vhci->devices[vhci->port_to_device[i]]->tq_mask & BIT(j))) -+ continue; -+ bce_vhci_transfer_queue_resume(&vhci->devices[vhci->port_to_device[i]]->tq[j], -+ BCE_VHCI_PAUSE_SUSPEND); -+ } -+ } -+ -+ pr_info("bce_vhci: resume done\n"); -+ return 0; -+} -+ -+static int bce_vhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags) -+{ -+ struct bce_vhci_transfer_queue *q = urb->ep->hcpriv; -+ pr_debug("bce_vhci_urb_enqueue %i:%x\n", q->dev_addr, urb->ep->desc.bEndpointAddress); -+ if (!q) -+ return -ENOENT; -+ return bce_vhci_urb_create(q, urb); -+} -+ -+static int bce_vhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status) -+{ -+ struct bce_vhci_transfer_queue *q = urb->ep->hcpriv; -+ pr_debug("bce_vhci_urb_dequeue %x\n", urb->ep->desc.bEndpointAddress); -+ return bce_vhci_urb_request_cancel(q, urb, status); -+} -+ -+static void bce_vhci_endpoint_reset(struct usb_hcd *hcd, struct usb_host_endpoint *ep) -+{ -+ struct bce_vhci_transfer_queue *q = ep->hcpriv; -+ pr_debug("bce_vhci_endpoint_reset\n"); -+ if (q) -+ bce_vhci_transfer_queue_request_reset(q); -+} -+ -+static u8 bce_vhci_endpoint_index(u8 addr) -+{ -+ if (addr & 0x80) -+ return (u8) (0x10 + (addr & 0xf)); -+ return (u8) (addr & 0xf); -+} -+ -+static int bce_vhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev, struct usb_host_endpoint *endp) -+{ -+ u8 endp_index = bce_vhci_endpoint_index(endp->desc.bEndpointAddress); -+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); -+ bce_vhci_device_t devid = vhci->port_to_device[udev->portnum]; -+ struct bce_vhci_device *vdev = vhci->devices[devid]; -+ pr_debug("bce_vhci_add_endpoint %x/%x:%x\n", udev->portnum, devid, endp_index); -+ -+ if (udev->bus->root_hub == udev) /* The USB hub */ -+ return 0; -+ if (vdev == NULL) -+ return -ENODEV; -+ if (vdev->tq_mask & BIT(endp_index)) { -+ endp->hcpriv = &vdev->tq[endp_index]; -+ return 0; -+ } -+ -+ bce_vhci_create_transfer_queue(vhci, &vdev->tq[endp_index], endp, devid, -+ usb_endpoint_dir_in(&endp->desc) ? DMA_FROM_DEVICE : DMA_TO_DEVICE); -+ endp->hcpriv = &vdev->tq[endp_index]; -+ vdev->tq_mask |= BIT(endp_index); -+ -+ bce_vhci_cmd_endpoint_create(&vhci->cq, devid, &endp->desc); -+ return 0; -+} -+ -+static int bce_vhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev, struct usb_host_endpoint *endp) -+{ -+ u8 endp_index = bce_vhci_endpoint_index(endp->desc.bEndpointAddress); -+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); -+ bce_vhci_device_t devid = vhci->port_to_device[udev->portnum]; -+ struct bce_vhci_transfer_queue *q = endp->hcpriv; -+ struct bce_vhci_device *vdev = vhci->devices[devid]; -+ pr_info("bce_vhci_drop_endpoint %x:%x\n", udev->portnum, endp_index); -+ if (!q) { -+ if (vdev && vdev->tq_mask & BIT(endp_index)) { -+ pr_err("something deleted the hcpriv?\n"); -+ q = &vdev->tq[endp_index]; -+ } else { -+ return 0; -+ } -+ } -+ -+ bce_vhci_cmd_endpoint_destroy(&vhci->cq, devid, (u8) (endp->desc.bEndpointAddress & 0x8Fu)); -+ vhci->devices[devid]->tq_mask &= ~BIT(endp_index); -+ bce_vhci_destroy_transfer_queue(vhci, q); -+ return 0; -+} -+ -+static int bce_vhci_create_message_queues(struct bce_vhci *vhci) -+{ -+ if (bce_vhci_message_queue_create(vhci, &vhci->msg_commands, "VHC1HostCommands") || -+ bce_vhci_message_queue_create(vhci, &vhci->msg_system, "VHC1HostSystemEvents") || -+ bce_vhci_message_queue_create(vhci, &vhci->msg_isochronous, "VHC1HostIsochronousEvents") || -+ bce_vhci_message_queue_create(vhci, &vhci->msg_interrupt, "VHC1HostInterruptEvents") || -+ bce_vhci_message_queue_create(vhci, &vhci->msg_asynchronous, "VHC1HostAsynchronousEvents")) { -+ bce_vhci_destroy_message_queues(vhci); -+ return -EINVAL; -+ } -+ spin_lock_init(&vhci->msg_asynchronous_lock); -+ bce_vhci_command_queue_create(&vhci->cq, &vhci->msg_commands); -+ return 0; -+} -+ -+static void bce_vhci_destroy_message_queues(struct bce_vhci *vhci) -+{ -+ bce_vhci_command_queue_destroy(&vhci->cq); -+ bce_vhci_message_queue_destroy(vhci, &vhci->msg_commands); -+ bce_vhci_message_queue_destroy(vhci, &vhci->msg_system); -+ bce_vhci_message_queue_destroy(vhci, &vhci->msg_isochronous); -+ bce_vhci_message_queue_destroy(vhci, &vhci->msg_interrupt); -+ bce_vhci_message_queue_destroy(vhci, &vhci->msg_asynchronous); -+} -+ -+static void bce_vhci_handle_system_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg); -+static void bce_vhci_handle_usb_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg); -+ -+static int bce_vhci_create_event_queues(struct bce_vhci *vhci) -+{ -+ vhci->ev_cq = bce_create_cq(vhci->dev, 0x100); -+ if (!vhci->ev_cq) -+ return -EINVAL; -+#define CREATE_EVENT_QUEUE(field, name, cb) bce_vhci_event_queue_create(vhci, &vhci->field, name, cb) -+ if (__bce_vhci_event_queue_create(vhci, &vhci->ev_commands, "VHC1FirmwareCommands", -+ bce_vhci_firmware_event_completion) || -+ CREATE_EVENT_QUEUE(ev_system, "VHC1FirmwareSystemEvents", bce_vhci_handle_system_event) || -+ CREATE_EVENT_QUEUE(ev_isochronous, "VHC1FirmwareIsochronousEvents", bce_vhci_handle_usb_event) || -+ CREATE_EVENT_QUEUE(ev_interrupt, "VHC1FirmwareInterruptEvents", bce_vhci_handle_usb_event) || -+ CREATE_EVENT_QUEUE(ev_asynchronous, "VHC1FirmwareAsynchronousEvents", bce_vhci_handle_usb_event)) { -+ bce_vhci_destroy_event_queues(vhci); -+ return -EINVAL; -+ } -+#undef CREATE_EVENT_QUEUE -+ return 0; -+} -+ -+static void bce_vhci_destroy_event_queues(struct bce_vhci *vhci) -+{ -+ bce_vhci_event_queue_destroy(vhci, &vhci->ev_commands); -+ bce_vhci_event_queue_destroy(vhci, &vhci->ev_system); -+ bce_vhci_event_queue_destroy(vhci, &vhci->ev_isochronous); -+ bce_vhci_event_queue_destroy(vhci, &vhci->ev_interrupt); -+ bce_vhci_event_queue_destroy(vhci, &vhci->ev_asynchronous); -+ if (vhci->ev_cq) -+ bce_destroy_cq(vhci->dev, vhci->ev_cq); -+} -+ -+static void bce_vhci_send_fw_event_response(struct bce_vhci *vhci, struct bce_vhci_message *req, u16 status) -+{ -+ unsigned long timeout = 1000; -+ struct bce_vhci_message r = *req; -+ r.cmd = (u16) (req->cmd | 0x8000u); -+ r.status = status; -+ r.param1 = req->param1; -+ r.param2 = 0; -+ -+ if (bce_reserve_submission(vhci->msg_system.sq, &timeout)) { -+ pr_err("bce-vhci: Cannot reserve submision for FW event reply\n"); -+ return; -+ } -+ bce_vhci_message_queue_write(&vhci->msg_system, &r); -+} -+ -+static int bce_vhci_handle_firmware_event(struct bce_vhci *vhci, struct bce_vhci_message *msg) -+{ -+ unsigned long flags; -+ bce_vhci_device_t devid; -+ u8 endp; -+ struct bce_vhci_device *dev; -+ struct bce_vhci_transfer_queue *tq; -+ if (msg->cmd == BCE_VHCI_CMD_ENDPOINT_REQUEST_STATE || msg->cmd == BCE_VHCI_CMD_ENDPOINT_SET_STATE) { -+ devid = (bce_vhci_device_t) (msg->param1 & 0xff); -+ endp = bce_vhci_endpoint_index((u8) ((msg->param1 >> 8) & 0xff)); -+ dev = vhci->devices[devid]; -+ if (!dev || !(dev->tq_mask & BIT(endp))) -+ return BCE_VHCI_BAD_ARGUMENT; -+ tq = &dev->tq[endp]; -+ } -+ -+ if (msg->cmd == BCE_VHCI_CMD_ENDPOINT_REQUEST_STATE) { -+ if (msg->param2 == BCE_VHCI_ENDPOINT_ACTIVE) { -+ bce_vhci_transfer_queue_resume(tq, BCE_VHCI_PAUSE_FIRMWARE); -+ return BCE_VHCI_SUCCESS; -+ } else if (msg->param2 == BCE_VHCI_ENDPOINT_PAUSED) { -+ bce_vhci_transfer_queue_pause(tq, BCE_VHCI_PAUSE_FIRMWARE); -+ return BCE_VHCI_SUCCESS; -+ } -+ return BCE_VHCI_BAD_ARGUMENT; -+ } else if (msg->cmd == BCE_VHCI_CMD_ENDPOINT_SET_STATE) { -+ if (msg->param2 == BCE_VHCI_ENDPOINT_STALLED) { -+ tq->state = msg->param2; -+ spin_lock_irqsave(&tq->urb_lock, flags); -+ tq->stalled = true; -+ spin_unlock_irqrestore(&tq->urb_lock, flags); -+ return BCE_VHCI_SUCCESS; -+ } -+ return BCE_VHCI_BAD_ARGUMENT; -+ } -+ pr_warn("bce-vhci: Unhandled firmware event: %x s=%x p1=%x p2=%llx\n", -+ msg->cmd, msg->status, msg->param1, msg->param2); -+ return BCE_VHCI_BAD_ARGUMENT; -+} -+ -+static void bce_vhci_handle_firmware_events_w(struct work_struct *ws) -+{ -+ size_t cnt = 0; -+ int result; -+ struct bce_vhci *vhci = container_of(ws, struct bce_vhci, w_fw_events); -+ struct bce_queue_sq *sq = vhci->ev_commands.sq; -+ struct bce_sq_completion_data *cq; -+ struct bce_vhci_message *msg, *msg2 = NULL; -+ -+ while (true) { -+ if (msg2) { -+ msg = msg2; -+ msg2 = NULL; -+ } else if ((cq = bce_next_completion(sq))) { -+ if (cq->status == BCE_COMPLETION_ABORTED) { -+ bce_notify_submission_complete(sq); -+ continue; -+ } -+ msg = &vhci->ev_commands.data[sq->head]; -+ } else { -+ break; -+ } -+ -+ pr_debug("bce-vhci: Got fw event: %x s=%x p1=%x p2=%llx\n", msg->cmd, msg->status, msg->param1, msg->param2); -+ if ((cq = bce_next_completion(sq))) { -+ msg2 = &vhci->ev_commands.data[(sq->head + 1) % sq->el_count]; -+ pr_debug("bce-vhci: Got second fw event: %x s=%x p1=%x p2=%llx\n", -+ msg->cmd, msg->status, msg->param1, msg->param2); -+ if (cq->status != BCE_COMPLETION_ABORTED && -+ msg2->cmd == (msg->cmd | 0x4000) && msg2->param1 == msg->param1) { -+ /* Take two elements */ -+ pr_debug("bce-vhci: Cancelled\n"); -+ bce_vhci_send_fw_event_response(vhci, msg, BCE_VHCI_ABORT); -+ -+ bce_notify_submission_complete(sq); -+ bce_notify_submission_complete(sq); -+ msg2 = NULL; -+ cnt += 2; -+ continue; -+ } -+ -+ pr_warn("bce-vhci: Handle fw event - unexpected cancellation\n"); -+ } -+ -+ result = bce_vhci_handle_firmware_event(vhci, msg); -+ bce_vhci_send_fw_event_response(vhci, msg, (u16) result); -+ -+ -+ bce_notify_submission_complete(sq); -+ ++cnt; -+ } -+ bce_vhci_event_queue_submit_pending(&vhci->ev_commands, cnt); -+ if (atomic_read(&sq->available_commands) == sq->el_count - 1) { -+ pr_debug("bce-vhci: complete\n"); -+ complete(&vhci->ev_commands.queue_empty_completion); -+ } -+} -+ -+static void bce_vhci_firmware_event_completion(struct bce_queue_sq *sq) -+{ -+ struct bce_vhci_event_queue *q = sq->userdata; -+ queue_work(q->vhci->tq_state_wq, &q->vhci->w_fw_events); -+} -+ -+static void bce_vhci_handle_system_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg) -+{ -+ if (msg->cmd & 0x8000) { -+ bce_vhci_command_queue_deliver_completion(&q->vhci->cq, msg); -+ } else { -+ pr_warn("bce-vhci: Unhandled system event: %x s=%x p1=%x p2=%llx\n", -+ msg->cmd, msg->status, msg->param1, msg->param2); -+ } -+} -+ -+static void bce_vhci_handle_usb_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg) -+{ -+ bce_vhci_device_t devid; -+ u8 endp; -+ struct bce_vhci_device *dev; -+ if (msg->cmd & 0x8000) { -+ bce_vhci_command_queue_deliver_completion(&q->vhci->cq, msg); -+ } else if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST || msg->cmd == BCE_VHCI_CMD_CONTROL_TRANSFER_STATUS) { -+ devid = (bce_vhci_device_t) (msg->param1 & 0xff); -+ endp = bce_vhci_endpoint_index((u8) ((msg->param1 >> 8) & 0xff)); -+ dev = q->vhci->devices[devid]; -+ if (!dev || (dev->tq_mask & BIT(endp)) == 0) { -+ pr_err("bce-vhci: Didn't find destination for transfer queue event\n"); -+ return; -+ } -+ bce_vhci_transfer_queue_event(&dev->tq[endp], msg); -+ } else { -+ pr_warn("bce-vhci: Unhandled USB event: %x s=%x p1=%x p2=%llx\n", -+ msg->cmd, msg->status, msg->param1, msg->param2); -+ } -+} -+ -+ -+ -+static const struct hc_driver bce_vhci_driver = { -+ .description = "bce-vhci", -+ .product_desc = "BCE VHCI Host Controller", -+ .hcd_priv_size = sizeof(struct bce_vhci *), -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0) -+ .flags = HCD_USB2, -+#else -+ .flags = HCD_USB2 | HCD_DMA, -+#endif -+ -+ .start = bce_vhci_start, -+ .stop = bce_vhci_stop, -+ .hub_status_data = bce_vhci_hub_status_data, -+ .hub_control = bce_vhci_hub_control, -+ .urb_enqueue = bce_vhci_urb_enqueue, -+ .urb_dequeue = bce_vhci_urb_dequeue, -+ .enable_device = bce_vhci_enable_device, -+ .free_dev = bce_vhci_free_device, -+ .address_device = bce_vhci_address_device, -+ .add_endpoint = bce_vhci_add_endpoint, -+ .drop_endpoint = bce_vhci_drop_endpoint, -+ .endpoint_reset = bce_vhci_endpoint_reset, -+ .check_bandwidth = bce_vhci_check_bandwidth, -+ .get_frame_number = bce_vhci_get_frame_number, -+ .bus_suspend = bce_vhci_bus_suspend, -+ .bus_resume = bce_vhci_bus_resume -+}; -+ -+ -+int __init bce_vhci_module_init(void) -+{ -+ int result; -+ if ((result = alloc_chrdev_region(&bce_vhci_chrdev, 0, 1, "bce-vhci"))) -+ goto fail_chrdev; -+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,4,0) -+ bce_vhci_class = class_create(THIS_MODULE, "bce-vhci"); -+#else -+ bce_vhci_class = class_create("bce-vhci"); -+#endif -+ if (IS_ERR(bce_vhci_class)) { -+ result = PTR_ERR(bce_vhci_class); -+ goto fail_class; -+ } -+ return 0; -+ -+fail_class: -+ class_destroy(bce_vhci_class); -+fail_chrdev: -+ unregister_chrdev_region(bce_vhci_chrdev, 1); -+ if (!result) -+ result = -EINVAL; -+ return result; -+} -+void __exit bce_vhci_module_exit(void) -+{ -+ class_destroy(bce_vhci_class); -+ unregister_chrdev_region(bce_vhci_chrdev, 1); -+} -+ -+module_param_named(vhci_port_mask, bce_vhci_port_mask, ushort, 0444); -+MODULE_PARM_DESC(vhci_port_mask, "Specifies which VHCI ports are enabled"); -diff --git a/drivers/staging/apple-bce/vhci/vhci.h b/drivers/staging/apple-bce/vhci/vhci.h -new file mode 100644 -index 000000000000..6c2e22622f4c ---- /dev/null -+++ b/drivers/staging/apple-bce/vhci/vhci.h -@@ -0,0 +1,52 @@ -+#ifndef BCE_VHCI_H -+#define BCE_VHCI_H -+ -+#include "queue.h" -+#include "transfer.h" -+ -+struct usb_hcd; -+struct bce_queue_cq; -+ -+struct bce_vhci_device { -+ struct bce_vhci_transfer_queue tq[32]; -+ u32 tq_mask; -+}; -+struct bce_vhci { -+ struct apple_bce_device *dev; -+ dev_t vdevt; -+ struct device *vdev; -+ struct usb_hcd *hcd; -+ struct spinlock hcd_spinlock; -+ struct bce_vhci_message_queue msg_commands; -+ struct bce_vhci_message_queue msg_system; -+ struct bce_vhci_message_queue msg_isochronous; -+ struct bce_vhci_message_queue msg_interrupt; -+ struct bce_vhci_message_queue msg_asynchronous; -+ struct spinlock msg_asynchronous_lock; -+ struct bce_vhci_command_queue cq; -+ struct bce_queue_cq *ev_cq; -+ struct bce_vhci_event_queue ev_commands; -+ struct bce_vhci_event_queue ev_system; -+ struct bce_vhci_event_queue ev_isochronous; -+ struct bce_vhci_event_queue ev_interrupt; -+ struct bce_vhci_event_queue ev_asynchronous; -+ u16 port_mask; -+ u8 port_count; -+ u16 port_power_mask; -+ bce_vhci_device_t port_to_device[16]; -+ struct bce_vhci_device *devices[16]; -+ struct workqueue_struct *tq_state_wq; -+ struct work_struct w_fw_events; -+}; -+ -+int __init bce_vhci_module_init(void); -+void __exit bce_vhci_module_exit(void); -+ -+int bce_vhci_create(struct apple_bce_device *dev, struct bce_vhci *vhci); -+void bce_vhci_destroy(struct bce_vhci *vhci); -+int bce_vhci_start(struct usb_hcd *hcd); -+void bce_vhci_stop(struct usb_hcd *hcd); -+ -+struct bce_vhci *bce_vhci_from_hcd(struct usb_hcd *hcd); -+ -+#endif //BCE_VHCI_H -diff --git a/include/drm/drm_format_helper.h b/include/drm/drm_format_helper.h -index 428d81afe215..aa1604d92c1a 100644 ---- a/include/drm/drm_format_helper.h -+++ b/include/drm/drm_format_helper.h -@@ -96,6 +96,9 @@ void drm_fb_xrgb8888_to_rgba5551(struct iosys_map *dst, const unsigned int *dst_ - void drm_fb_xrgb8888_to_rgb888(struct iosys_map *dst, const unsigned int *dst_pitch, - const struct iosys_map *src, const struct drm_framebuffer *fb, - const struct drm_rect *clip, struct drm_format_conv_state *state); -+void drm_fb_xrgb8888_to_bgr888(struct iosys_map *dst, const unsigned int *dst_pitch, -+ const struct iosys_map *src, const struct drm_framebuffer *fb, -+ const struct drm_rect *clip, struct drm_format_conv_state *state); - void drm_fb_xrgb8888_to_argb8888(struct iosys_map *dst, const unsigned int *dst_pitch, - const struct iosys_map *src, const struct drm_framebuffer *fb, - const struct drm_rect *clip, struct drm_format_conv_state *state); -diff --git a/lib/test_printf.c b/lib/test_printf.c -index 59dbe4f9a4cb..6fc82cb0b4cf 100644 ---- a/lib/test_printf.c -+++ b/lib/test_printf.c -@@ -779,18 +779,26 @@ static void __init fwnode_pointer(void) - static void __init fourcc_pointer(void) - { - struct { -+ char type; - u32 code; - char *str; - } const try[] = { -- { 0x3231564e, "NV12 little-endian (0x3231564e)", }, -- { 0xb231564e, "NV12 big-endian (0xb231564e)", }, -- { 0x10111213, ".... little-endian (0x10111213)", }, -- { 0x20303159, "Y10 little-endian (0x20303159)", }, -+ { 'c', 0x3231564e, "NV12 little-endian (0x3231564e)", }, -+ { 'c', 0xb231564e, "NV12 big-endian (0xb231564e)", }, -+ { 'c', 0x10111213, ".... little-endian (0x10111213)", }, -+ { 'c', 0x20303159, "Y10 little-endian (0x20303159)", }, -+ { 'h', 0x67503030, "gP00 (0x67503030)", }, -+ { 'r', 0x30305067, "gP00 (0x67503030)", }, -+ { 'l', cpu_to_le32(0x67503030), "gP00 (0x67503030)", }, -+ { 'b', cpu_to_be32(0x67503030), "gP00 (0x67503030)", }, - }; - unsigned int i; - -- for (i = 0; i < ARRAY_SIZE(try); i++) -- test(try[i].str, "%p4cc", &try[i].code); -+ for (i = 0; i < ARRAY_SIZE(try); i++) { -+ char fmt[] = { '%', 'p', '4', 'c', try[i].type, '\0' }; -+ -+ test(try[i].str, fmt, &try[i].code); -+ } - } - - static void __init -diff --git a/lib/vsprintf.c b/lib/vsprintf.c -index 9d3dac38a3f4..17926ad7863c 100644 ---- a/lib/vsprintf.c -+++ b/lib/vsprintf.c -@@ -1795,27 +1795,50 @@ char *fourcc_string(char *buf, char *end, const u32 *fourcc, - char output[sizeof("0123 little-endian (0x01234567)")]; - char *p = output; - unsigned int i; -+ bool pix_fmt = false; - u32 orig, val; - -- if (fmt[1] != 'c' || fmt[2] != 'c') -+ if (fmt[1] != 'c') - return error_string(buf, end, "(%p4?)", spec); - - if (check_pointer(&buf, end, fourcc, spec)) - return buf; - - orig = get_unaligned(fourcc); -- val = orig & ~BIT(31); -+ switch (fmt[2]) { -+ case 'h': -+ val = orig; -+ break; -+ case 'r': -+ val = orig = swab32(orig); -+ break; -+ case 'l': -+ val = orig = le32_to_cpu(orig); -+ break; -+ case 'b': -+ val = orig = be32_to_cpu(orig); -+ break; -+ case 'c': -+ /* Pixel formats are printed LSB-first */ -+ val = swab32(orig & ~BIT(31)); -+ pix_fmt = true; -+ break; -+ default: -+ return error_string(buf, end, "(%p4?)", spec); -+ } - - for (i = 0; i < sizeof(u32); i++) { -- unsigned char c = val >> (i * 8); -+ unsigned char c = val >> ((3 - i) * 8); - - /* Print non-control ASCII characters as-is, dot otherwise */ - *p++ = isascii(c) && isprint(c) ? c : '.'; - } - -- *p++ = ' '; -- strcpy(p, orig & BIT(31) ? "big-endian" : "little-endian"); -- p += strlen(p); -+ if (pix_fmt) { -+ *p++ = ' '; -+ strcpy(p, orig & BIT(31) ? "big-endian" : "little-endian"); -+ p += strlen(p); -+ } - - *p++ = ' '; - *p++ = '('; -@@ -2379,6 +2402,7 @@ char *rust_fmt_argument(char *buf, char *end, void *ptr); - * read the documentation (path below) first. - * - 'NF' For a netdev_features_t - * - '4cc' V4L2 or DRM FourCC code, with endianness and raw numerical value. -+ * - '4c[hlbr]' Generic FourCC code. - * - 'h[CDN]' For a variable-length buffer, it prints it as a hex string with - * a certain separator (' ' by default): - * C colon -diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl -index 9eed3683ad76..7ddbf75f4c26 100755 ---- a/scripts/checkpatch.pl -+++ b/scripts/checkpatch.pl -@@ -6912,7 +6912,7 @@ sub process { - ($extension eq "f" && - defined $qualifier && $qualifier !~ /^w/) || - ($extension eq "4" && -- defined $qualifier && $qualifier !~ /^cc/)) { -+ defined $qualifier && $qualifier !~ /^c[chlbr]/)) { - $bad_specifier = $specifier; - last; - } --- -2.48.0.rc1 - -From 7d65948abcfc815f56268ab3d7a90d175520536d Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Sun, 2 Feb 2025 13:55:24 +0100 -Subject: [PATCH 12/12] zstd - -Signed-off-by: Peter Jung +Signed-off-by: Eric Naim --- include/linux/zstd.h | 2 +- include/linux/zstd_errors.h | 23 +- @@ -48973,4 +31384,4 @@ index 469fc3059be0..0ae819f0c927 100644 EXPORT_SYMBOL(zstd_reset_dstream); -- -2.48.0.rc1 +2.48.1