From 42aacec658a804ffbd46535125897c2a089fb55d Mon Sep 17 00:00:00 2001 From: zhangyuyang Date: Fri, 24 Oct 2025 08:07:57 +0800 Subject: [PATCH] drivers:misc:sdma-dae: optimize kernel code kunpeng inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/ID3BBR CVE: NA ---------------------------------------------------------------------- 1. sdma exit logic optimize 2. sdma pin memory optimize 3. sdma authority table optimize Fixes: f8eeb3987916 ("drivers: misc: sdma-dae: support channel management") Signed-off-by: zhangyuyang --- drivers/misc/sdma-dae/hisi_sdma.h | 3 +- drivers/misc/sdma-dae/sdma_auth.c | 31 +-- drivers/misc/sdma-dae/sdma_cdev.c | 430 +++++++++++++++--------------- drivers/misc/sdma-dae/sdma_dbg.c | 2 +- drivers/misc/sdma-dae/sdma_hal.h | 75 +++--- drivers/misc/sdma-dae/sdma_irq.c | 4 +- drivers/misc/sdma-dae/sdma_main.c | 65 +++-- drivers/misc/sdma-dae/sdma_umem.c | 63 +++-- drivers/misc/sdma-dae/sdma_umem.h | 3 +- 9 files changed, 340 insertions(+), 336 deletions(-) diff --git a/drivers/misc/sdma-dae/hisi_sdma.h b/drivers/misc/sdma-dae/hisi_sdma.h index c5cc92de5ffc..7fae1621651e 100644 --- a/drivers/misc/sdma-dae/hisi_sdma.h +++ b/drivers/misc/sdma-dae/hisi_sdma.h @@ -26,8 +26,9 @@ #define HISI_SDMA_SQ_LENGTH (1U << 10) #define HISI_SDMA_CQ_LENGTH (1U << 10) +#define HISI_SDMA_MAX_CHN_NUM 192 #define HISI_STARS_CHN_NUM 32 -#define HISI_SDMA_DEFAULT_CHANNEL_NUM (192 - HISI_STARS_CHN_NUM) +#define HISI_SDMA_DEFAULT_CHANNEL_NUM (HISI_SDMA_MAX_CHN_NUM - HISI_STARS_CHN_NUM) #define HISI_SDMA_SQ_SIZE (HISI_SDMA_SQ_ENTRY_SIZE * HISI_SDMA_SQ_LENGTH) #define HISI_SDMA_CQ_SIZE (HISI_SDMA_CQ_ENTRY_SIZE * HISI_SDMA_CQ_LENGTH) #define HISI_SDMA_REG_SIZE 4096 diff --git a/drivers/misc/sdma-dae/sdma_auth.c b/drivers/misc/sdma-dae/sdma_auth.c index e15a7d4b96ce..1cfbc6fe1c7b 100644 --- a/drivers/misc/sdma-dae/sdma_auth.c +++ b/drivers/misc/sdma-dae/sdma_auth.c @@ -113,20 +113,16 @@ static struct hisi_sdma_own_pid_hte *sdma_search_owner_pid_ht(u32 pid) return NULL; } -static void sdma_clear_residual_auth_ht(struct hisi_sdma_own_pid_hte *entry, u32 *list, u32 pos, - bool *stored_info) +static void sdma_clear_residual_auth_ht(struct hisi_sdma_own_pid_hte *entry, u32 *list, u32 pos) { struct hisi_sdma_sub_pid_hte *sub_entry; - u32 i = pos; + u32 i; - while (i > 0) { - i--; - if (stored_info[i]) { - sub_entry = sdma_search_submitter_pid(entry, list[i]); - if (sub_entry) { - hash_del(&sub_entry->pnode); - kfree(sub_entry); - } + for (i = 0; i < pos; i++) { + sub_entry = sdma_search_submitter_pid(entry, list[i]); + if (sub_entry) { + hash_del(&sub_entry->pnode); + kfree(sub_entry); } } } @@ -134,12 +130,7 @@ static void sdma_clear_residual_auth_ht(struct hisi_sdma_own_pid_hte *entry, u32 static int sdma_add_authority_ht(struct hisi_sdma_own_pid_hte *entry, u32 count, u32 *list) { struct hisi_sdma_sub_pid_hte *sub_entry; - bool *stored; - u32 i; - - stored = kcalloc(count, sizeof(bool), GFP_ATOMIC); - if (!stored) - return -ENOMEM; + u32 i, j = 0; for (i = 0; i < count; i++) { sub_entry = sdma_search_submitter_pid(entry, list[i]); @@ -148,17 +139,15 @@ static int sdma_add_authority_ht(struct hisi_sdma_own_pid_hte *entry, u32 count, sub_entry = kzalloc(sizeof(struct hisi_sdma_sub_pid_hte), GFP_ATOMIC); if (!sub_entry) { - sdma_clear_residual_auth_ht(entry, list, i, stored); - kfree(stored); + sdma_clear_residual_auth_ht(entry, list, j); return -ENOMEM; } sub_entry->pid = list[i]; hash_add(entry->sdma_submitter_pid_ht, &sub_entry->pnode, sub_entry->pid); - stored[i] = true; + list[j++] = list[i]; } - kfree(stored); return 0; } diff --git a/drivers/misc/sdma-dae/sdma_cdev.c b/drivers/misc/sdma-dae/sdma_cdev.c index efa942ef2588..dc3057fc0a71 100644 --- a/drivers/misc/sdma-dae/sdma_cdev.c +++ b/drivers/misc/sdma-dae/sdma_cdev.c @@ -8,6 +8,7 @@ #include #include #include +#include #include "sdma_hal.h" #include "sdma_umem.h" @@ -15,10 +16,9 @@ #define mn_to_sdma(mn) container_of(mn, struct hisi_sdma_mn, mn) -static struct hisi_sdma_global_info g_info; +DEFINE_MUTEX(sdma_pid_lock); -static atomic_t ttl_processes; -static atomic_t exit_processes; +static struct hisi_sdma_global_info g_info; struct hisi_sdma_channel_list { struct list_head chn_list; @@ -36,7 +36,6 @@ struct file_open_data { }; struct hisi_sdma_mn { - uint32_t pid; refcount_t refs; struct mmu_notifier mn; struct file_open_data *data; @@ -53,12 +52,11 @@ struct pasid_info { u32 dst_pasid; }; -static struct hisi_sdma_pid_ref_hte *sdma_search_pid_ref(struct hisi_sdma_device *psdma_dev, - u32 pid) +static struct hisi_sdma_pid_ref_hte *sdma_search_pid_ref(u32 pid) { struct hisi_sdma_pid_ref_hte *entry = NULL; - hash_for_each_possible(psdma_dev->sdma_pid_ref_ht, entry, node, pid) { + hash_for_each_possible(g_info.sdma_pid_ref_ht, entry, node, pid) { if (entry->pid == pid) return entry; } @@ -66,36 +64,35 @@ static struct hisi_sdma_pid_ref_hte *sdma_search_pid_ref(struct hisi_sdma_device return NULL; } -static int sdma_add_pid_ref(struct hisi_sdma_device *psdma_dev, u32 pid) +static int sdma_add_pid_ref(u32 pid, u16 node_idx) { struct hisi_sdma_pid_ref_hte *entry = NULL; - spin_lock(&psdma_dev->pid_lock); - entry = sdma_search_pid_ref(psdma_dev, pid); + mutex_lock(&sdma_pid_lock); + entry = sdma_search_pid_ref(pid); if (!entry) { - entry = kmalloc_node(sizeof(struct hisi_sdma_pid_ref_hte), GFP_ATOMIC, - psdma_dev->node_idx); + entry = kmalloc_node(sizeof(struct hisi_sdma_pid_ref_hte), GFP_ATOMIC, node_idx); if (!entry) { - spin_unlock(&psdma_dev->pid_lock); + mutex_unlock(&sdma_pid_lock); return -ENOMEM; } entry->pid = pid; entry->ref = 1; - hash_add(psdma_dev->sdma_pid_ref_ht, &entry->node, entry->pid); - } else { + hash_add(g_info.sdma_pid_ref_ht, &entry->node, entry->pid); + } else entry->ref++; - } - spin_unlock(&psdma_dev->pid_lock); + + mutex_unlock(&sdma_pid_lock); return 0; } -static void sdma_del_pid_ref(struct hisi_sdma_device *psdma_dev, u32 pid) +static void sdma_del_pid_ref(u32 pid) { struct hisi_sdma_pid_ref_hte *entry = NULL; - spin_lock(&psdma_dev->pid_lock); - entry = sdma_search_pid_ref(psdma_dev, pid); + mutex_lock(&sdma_pid_lock); + entry = sdma_search_pid_ref(pid); if (entry) { entry->ref--; if (entry->ref == 0) { @@ -104,21 +101,21 @@ static void sdma_del_pid_ref(struct hisi_sdma_device *psdma_dev, u32 pid) sdma_free_authority_ht_with_pid(pid); } } - spin_unlock(&psdma_dev->pid_lock); + mutex_unlock(&sdma_pid_lock); } -void sdma_clear_pid_ref(struct hisi_sdma_device *psdma_dev) +void sdma_clear_pid_ref(void) { struct hisi_sdma_pid_ref_hte *entry = NULL; struct hlist_node *tmp; u32 bkt; - spin_lock(&psdma_dev->pid_lock); - hash_for_each_safe(psdma_dev->sdma_pid_ref_ht, bkt, tmp, entry, node) { + mutex_lock(&sdma_pid_lock); + hash_for_each_safe(g_info.sdma_pid_ref_ht, bkt, tmp, entry, node) { hash_del(&entry->node); kfree(entry); } - spin_unlock(&psdma_dev->pid_lock); + mutex_unlock(&sdma_pid_lock); } static struct hisi_sdma_channel_node *sdma_search_ida_ref(struct hisi_sdma_channel *pchannel, @@ -130,6 +127,7 @@ static struct hisi_sdma_channel_node *sdma_search_ida_ref(struct hisi_sdma_chann if (entry->ida == ida) return entry; } + return NULL; } @@ -151,6 +149,7 @@ static int sdma_add_ida_ref(struct hisi_sdma_channel *pchannel, int ida) } else entry->ref++; spin_unlock(&pchannel->owner_chn_lock); + return 0; } @@ -171,6 +170,7 @@ static bool sdma_del_ida_ref(struct hisi_sdma_channel *pchannel, int ida) return false; } spin_unlock(&pchannel->owner_chn_lock); + return true; } @@ -196,6 +196,7 @@ static bool sdma_check_ida_val(struct hisi_sdma_channel *pchannel, int ida) return false; } spin_unlock(&pchannel->owner_chn_lock); + return true; } @@ -216,54 +217,37 @@ void sdma_clear_ida_ref(struct hisi_sdma_channel *pchannel) spin_unlock(&pchannel->owner_chn_lock); } -static bool sdma_wait_hardware_done(struct hisi_sdma_channel *pchannel) +static int sdma_wait_cq_writeback(struct hisi_sdma_channel *pchannel) { - u32 sq_tail, sq_head; - u32 cnt = 0; - - sq_head = sdma_channel_get_sq_head(pchannel); - sq_tail = sdma_channel_get_sq_tail(pchannel); - while (sq_head != sq_tail && cnt <= SDMA_POLL_TIMEOUT) { - sq_head = sdma_channel_get_sq_head(pchannel); - sq_tail = sdma_channel_get_sq_tail(pchannel); - cnt++; - msleep(SDMA_POLL_DELAY); - } - - return (cnt <= SDMA_POLL_TIMEOUT); -} - -static bool sdma_wait_cq_writeback(struct hisi_sdma_channel *pchannel) -{ - u32 cq_tail, sq_tail; + u32 cq_tail; u32 cnt = 0; - cq_tail = sdma_channel_get_cq_tail(pchannel); - sq_tail = sdma_channel_get_sq_tail(pchannel); - while (cq_tail != sq_tail && cnt <= SDMA_POLL_TIMEOUT) { + while (!sdma_channel_is_quiescent(pchannel) && cnt++ < SDMA_POLL_TIMEOUT) { cq_tail = sdma_channel_get_cq_tail(pchannel); - cnt++; + sdma_channel_set_cq_head(pchannel, cq_tail); + pchannel->sync_info_base->cq_head = (u16)cq_tail; + pchannel->sync_info_base->cq_tail = (u16)cq_tail; + pchannel->sync_info_base->sq_head = (u16)cq_tail; msleep(SDMA_POLL_DELAY); } - return (cnt <= SDMA_POLL_TIMEOUT); + if (cnt > SDMA_POLL_TIMEOUT) + return -ETIMEDOUT; + + return 0; } static void sdma_pause_single_channel(struct hisi_sdma_channel *pchannel, struct hisi_sdma_device *psdma_dev) { - u16 idx = pchannel->idx; - - if (!sdma_wait_hardware_done(pchannel)) - pr_warn("SDMA %u chn %hu hardware not finish all sqes!\n", - psdma_dev->idx, idx); + int ret; sdma_channel_set_pause(pchannel); - if (sdma_wait_cq_writeback(pchannel)) - sdma_channel_reset_sq_cq(pchannel); - else - pr_warn("SDMA %u chn %hu hardware not write back all cqes!\n", - psdma_dev->idx, idx); + + ret = sdma_wait_cq_writeback(pchannel); + if (ret < 0) + pr_warn("Task unfinished for sdma %u channel %u, please wait.\n", + psdma_dev->idx, pchannel->idx); } static void sdma_pause_channels(struct hisi_sdma_device *psdma_dev) @@ -271,93 +255,74 @@ static void sdma_pause_channels(struct hisi_sdma_device *psdma_dev) struct hisi_sdma_channel *pchannel; int i; + pr_warn("Process exit abnormally. Pause sdma%u proactively.\n", psdma_dev->idx); for (i = 0; i < HISI_SDMA_DEFAULT_CHANNEL_NUM; i++) { pchannel = psdma_dev->channels + i; sdma_pause_single_channel(pchannel, psdma_dev); } } -static void sdma_wait_channel_quiescent(struct hisi_sdma_device *psdma_dev) +static void sdma_mmu_release_pause(struct mmu_notifier *mn, struct mm_struct *mm) { - struct hisi_sdma_channel *pchannel; - int i; + struct hisi_sdma_device *psdma_dev; + struct hisi_sdma_mn *sdma_mn; + struct file_open_data *data; + u64 start_jiffies; + int ref_cnt; - for (i = 0; i < HISI_SDMA_DEFAULT_CHANNEL_NUM; i++) { - pchannel = psdma_dev->channels + i; - if (sdma_channel_is_quiescent(pchannel)) - continue; + sdma_mn = mn_to_sdma(mn); + data = sdma_mn->data; + if (!data) + return; - if (sdma_wait_cq_writeback(pchannel)) - sdma_channel_reset_sq_cq(pchannel); - else - pr_warn("SDMA %u chn %d hardware not write back all cqes!\n", - psdma_dev->idx, i); + psdma_dev = data->psdma_dev; + ref_cnt = refcount_read(&sdma_mn->refs); + atomic_sub(ref_cnt, &psdma_dev->sdma_dev_ref_cnt); + + if (atomic_inc_return(&psdma_dev->dev_exit_ref_cnt) == 1) { + sdma_pause_channels(psdma_dev); + atomic_set(&psdma_dev->dev_exit_flag, 1); + } else { + start_jiffies = get_jiffies_64(); + while (atomic_read(&psdma_dev->dev_exit_flag) != 1) { + if (time_is_before_jiffies64((u64)(start_jiffies + HZ))) + break; + cond_resched(); + } } } -static void sdma_resume_channel(struct hisi_sdma_device *psdma_dev) +static void sdma_resume_device(struct hisi_sdma_device *psdma_dev) { struct hisi_sdma_channel *pchannel; int i; + pr_warn("Resume sdma%u now.\n", psdma_dev->idx); for (i = 0; i < HISI_SDMA_DEFAULT_CHANNEL_NUM; i++) { pchannel = psdma_dev->channels + i; - if (!sdma_channel_is_paused(pchannel)) { - pr_warn("SDMA %u chn %d not paused\n", psdma_dev->idx, i); - continue; - } - if (!sdma_channel_is_quiescent(pchannel)) - sdma_channel_reset_sq_cq(pchannel); if (sdma_channel_is_paused(pchannel) && sdma_channel_is_quiescent(pchannel)) sdma_channel_write_resume(pchannel); } } -static void sdma_mmu_release_pause(struct mmu_notifier *mn, struct mm_struct *mm) -{ - struct hisi_sdma_device *psdma_dev; - struct hisi_sdma_mn *sdma_mn; - int i; - - sdma_mn = mn_to_sdma(mn); - if (!sdma_mn->data) - return; - - if (atomic_read(&exit_processes) == 0) { - atomic_set(&exit_processes, 1); - pr_warn("SDMA exit exceptionally, stop SDMA tasks before mm exit.\n"); - for (i = 0; i < g_info.core_dev->sdma_device_num; i++) { - psdma_dev = g_info.core_dev->sdma_devices[i]; - sdma_pause_channels(psdma_dev); - } - } else { - for (i = 0; i < g_info.core_dev->sdma_device_num; i++) { - psdma_dev = g_info.core_dev->sdma_devices[i]; - sdma_wait_channel_quiescent(psdma_dev); - } - } -} - static void sdma_mmu_release_resume(struct mmu_notifier *mn, struct mm_struct *mm) { struct hisi_sdma_device *psdma_dev; struct hisi_sdma_mn *sdma_mn; - int refcount; - int i; + struct file_open_data *data; + int ref_cnt; sdma_mn = mn_to_sdma(mn); - refcount = refcount_read(&sdma_mn->refs); - atomic_sub(refcount, &ttl_processes); - if (!sdma_mn->data) + data = sdma_mn->data; + if (!data) return; - if (atomic_read(&ttl_processes) == 0) { - pr_warn("Now resume SDMA channels after mm exit\n"); - for (i = 0; i < g_info.core_dev->sdma_device_num; i++) { - psdma_dev = g_info.core_dev->sdma_devices[i]; - sdma_resume_channel(psdma_dev); - } - atomic_set(&exit_processes, 0); + psdma_dev = data->psdma_dev; + ref_cnt = refcount_read(&sdma_mn->refs); + if (atomic_sub_return(ref_cnt, &psdma_dev->sdma_dev_ref_cnt) == 0) { + sdma_resume_device(psdma_dev); + atomic_set(&psdma_dev->dev_exit_ref_cnt, 0); + atomic_set(&psdma_dev->dev_exit_flag, 0); } } @@ -376,11 +341,12 @@ static const struct mmu_notifier_ops sdma_resume_mmu_notifier_ops = { .free_notifier = sdma_mmu_notifier_free, }; -static struct hisi_sdma_mn *search_resume_mmu_notifier(struct mm_struct *mm) +static struct hisi_sdma_mn *search_resume_mmu_notifier(struct mm_struct *mm, + struct hisi_sdma_device *psdma_dev) { struct hisi_sdma_mn *sdma_mn; - list_for_each_entry(sdma_mn, &g_info.sdma_resume_mm_list, list) { + list_for_each_entry(sdma_mn, &psdma_dev->sdma_resume_mm_list, list) { if (sdma_mn->mn.mm == mm) { refcount_inc(&sdma_mn->refs); return sdma_mn; @@ -390,11 +356,12 @@ static struct hisi_sdma_mn *search_resume_mmu_notifier(struct mm_struct *mm) return NULL; } -static struct hisi_sdma_mn *search_pause_mmu_notifier(struct mm_struct *mm) +static struct hisi_sdma_mn *search_pause_mmu_notifier(struct mm_struct *mm, + struct hisi_sdma_device *psdma_dev) { struct hisi_sdma_mn *sdma_mn; - list_for_each_entry(sdma_mn, &g_info.sdma_pause_mm_list, list) { + list_for_each_entry(sdma_mn, &psdma_dev->sdma_pause_mm_list, list) { if (sdma_mn->mn.mm == mm) { refcount_inc(&sdma_mn->refs); return sdma_mn; @@ -404,72 +371,86 @@ static struct hisi_sdma_mn *search_pause_mmu_notifier(struct mm_struct *mm) return NULL; } -static int sdma_pause_mmu_handler(struct mm_struct *mm, struct file_open_data *data) +static int sdma_pause_mmu_handler(struct file_open_data *data) { + struct hisi_sdma_device *psdma_dev = data->psdma_dev; struct hisi_sdma_mn *sdma_mn; int ret = 0; - mutex_lock(g_info.mutex_lock); - sdma_mn = search_pause_mmu_notifier(mm); + atomic_inc(&psdma_dev->sdma_dev_ref_cnt); + + mutex_lock(&psdma_dev->mutex_lock); + sdma_mn = search_pause_mmu_notifier(data->cur_file_mm, psdma_dev); if (sdma_mn) { - mutex_unlock(g_info.mutex_lock); + mutex_unlock(&psdma_dev->mutex_lock); return ret; } sdma_mn = kzalloc(sizeof(*sdma_mn), GFP_KERNEL); if (!sdma_mn) { - mutex_unlock(g_info.mutex_lock); - return -ENOMEM; + ret = -ENOMEM; + goto pause_unlock; } refcount_set(&sdma_mn->refs, 1); - sdma_mn->pid = current->tgid; sdma_mn->data = data; sdma_mn->mn.ops = &sdma_pause_mmu_notifier_ops; - ret = mmu_notifier_register(&sdma_mn->mn, mm); - if (ret) { - mutex_unlock(g_info.mutex_lock); - kfree(sdma_mn); - return ret; - } + ret = mmu_notifier_register(&sdma_mn->mn, data->cur_file_mm); + if (ret) + goto pause_free; + + list_add(&sdma_mn->list, &psdma_dev->sdma_pause_mm_list); + mutex_unlock(&psdma_dev->mutex_lock); + + return ret; - list_add(&sdma_mn->list, &g_info.sdma_pause_mm_list); - mutex_unlock(g_info.mutex_lock); +pause_free: + kfree(sdma_mn); +pause_unlock: + mutex_unlock(&psdma_dev->mutex_lock); + atomic_dec(&psdma_dev->sdma_dev_ref_cnt); return ret; } -static int sdma_resume_mmu_handler(struct mm_struct *mm, struct file_open_data *data) +static int sdma_resume_mmu_handler(struct file_open_data *data) { + struct hisi_sdma_device *psdma_dev = data->psdma_dev; struct hisi_sdma_mn *sdma_mn; int ret = 0; - mutex_lock(g_info.mutex_lock); - sdma_mn = search_resume_mmu_notifier(mm); + atomic_inc(&psdma_dev->sdma_dev_ref_cnt); + + mutex_lock(&psdma_dev->mutex_lock); + sdma_mn = search_resume_mmu_notifier(data->cur_file_mm, psdma_dev); if (sdma_mn) { - mutex_unlock(g_info.mutex_lock); + mutex_unlock(&psdma_dev->mutex_lock); return ret; } sdma_mn = kzalloc(sizeof(*sdma_mn), GFP_KERNEL); if (!sdma_mn) { - mutex_unlock(g_info.mutex_lock); - return -ENOMEM; + ret = -ENOMEM; + goto resume_unlock; } refcount_set(&sdma_mn->refs, 1); - sdma_mn->pid = current->tgid; sdma_mn->data = data; sdma_mn->mn.ops = &sdma_resume_mmu_notifier_ops; - ret = mmu_notifier_register(&sdma_mn->mn, mm); - if (ret) { - mutex_unlock(g_info.mutex_lock); - kfree(sdma_mn); - return ret; - } + ret = mmu_notifier_register(&sdma_mn->mn, data->cur_file_mm); + if (ret) + goto resume_free; + + list_add(&sdma_mn->list, &psdma_dev->sdma_resume_mm_list); + mutex_unlock(&psdma_dev->mutex_lock); + + return ret; - list_add(&sdma_mn->list, &g_info.sdma_resume_mm_list); - mutex_unlock(g_info.mutex_lock); +resume_free: + kfree(sdma_mn); +resume_unlock: + mutex_unlock(&psdma_dev->mutex_lock); + atomic_dec(&psdma_dev->sdma_dev_ref_cnt); return ret; } @@ -483,38 +464,43 @@ static void sdma_put_mmu_notifier(struct hisi_sdma_mn *sdma_mn) mmu_notifier_put(&sdma_mn->mn); } -static void sdma_put_resume_mmu_notifier(void) +static void sdma_put_resume_mmu_notifier(struct hisi_sdma_device *psdma_dev) { + struct mm_struct *mm = current->mm; struct hisi_sdma_mn *sdma_mn; - int pid = current->tgid; - mutex_lock(g_info.mutex_lock); - list_for_each_entry(sdma_mn, &g_info.sdma_resume_mm_list, list) { - if (sdma_mn->pid == pid) { + atomic_dec(&psdma_dev->sdma_dev_ref_cnt); + + mutex_lock(&psdma_dev->mutex_lock); + list_for_each_entry(sdma_mn, &psdma_dev->sdma_resume_mm_list, list) { + if (sdma_mn->mn.mm == mm) { sdma_put_mmu_notifier(sdma_mn); break; } } - mutex_unlock(g_info.mutex_lock); + mutex_unlock(&psdma_dev->mutex_lock); } -static void sdma_put_pause_mmu_notifier(void) +static void sdma_put_pause_mmu_notifier(struct hisi_sdma_device *psdma_dev) { + struct mm_struct *mm = current->mm; struct hisi_sdma_mn *sdma_mn; - int pid = current->tgid; - mutex_lock(g_info.mutex_lock); - list_for_each_entry(sdma_mn, &g_info.sdma_pause_mm_list, list) { - if (sdma_mn->pid == pid) { + atomic_dec(&psdma_dev->sdma_dev_ref_cnt); + + mutex_lock(&psdma_dev->mutex_lock); + list_for_each_entry(sdma_mn, &psdma_dev->sdma_pause_mm_list, list) { + if (sdma_mn->mn.mm == mm) { sdma_put_mmu_notifier(sdma_mn); break; } } - mutex_unlock(g_info.mutex_lock); + mutex_unlock(&psdma_dev->mutex_lock); } static int __do_sdma_open(struct hisi_sdma_device *psdma_dev, struct file *file) { + struct mm_struct *mm = current->mm; struct file_open_data *data; struct iommu_sva *handle; int id, ret; @@ -524,7 +510,7 @@ static int __do_sdma_open(struct hisi_sdma_device *psdma_dev, struct file *file) if (id < 0) return id; - ret = sdma_add_pid_ref(psdma_dev, (u32)current->tgid); + ret = sdma_add_pid_ref((u32)current->tgid, psdma_dev->node_idx); if (ret != 0) { dev_err(&psdma_dev->pdev->dev, "Alloc pid_ref hash failed\n"); goto free_ida; @@ -536,12 +522,14 @@ static int __do_sdma_open(struct hisi_sdma_device *psdma_dev, struct file *file) ret = -ENOMEM; goto free_pid_ref_ht; } + data->psdma_dev = psdma_dev; + data->cur_file_mm = mm; - ret = sdma_resume_mmu_handler(current->mm, data); + ret = sdma_resume_mmu_handler(data); if (ret != 0) goto free_privt_data; - handle = iommu_sva_bind_device(&psdma_dev->pdev->dev, current->mm, NULL); + handle = iommu_sva_bind_device(&psdma_dev->pdev->dev, mm, NULL); if (IS_ERR(handle)) { dev_err(&psdma_dev->pdev->dev, "Failed to bind sva, %ld\n", PTR_ERR(handle)); ret = (int)PTR_ERR(handle); @@ -554,16 +542,12 @@ static int __do_sdma_open(struct hisi_sdma_device *psdma_dev, struct file *file) goto sva_unbind; } - ret = sdma_pause_mmu_handler(current->mm, data); + ret = sdma_pause_mmu_handler(data); if (ret != 0) goto sva_unbind; - atomic_add(1, &ttl_processes); - - data->cur_file_mm = current->mm; data->ida = id; data->pasid = pasid; - data->psdma_dev = psdma_dev; data->handle = handle; INIT_LIST_HEAD(&data->non_share_chn_list); INIT_LIST_HEAD(&data->share_chn_list); @@ -575,13 +559,14 @@ static int __do_sdma_open(struct hisi_sdma_device *psdma_dev, struct file *file) sva_unbind: iommu_sva_unbind_device(handle); mmu_resume_unreg: - sdma_put_resume_mmu_notifier(); + sdma_put_resume_mmu_notifier(psdma_dev); free_privt_data: kfree(data); free_pid_ref_ht: - sdma_del_pid_ref(psdma_dev, current->tgid); + sdma_del_pid_ref(current->tgid); free_ida: ida_free(g_info.fd_ida, id); + return ret; } @@ -594,6 +579,7 @@ static int ioctl_get_sdma_num(struct file *file, unsigned long arg) return 0; } + static int ioctl_sdma_unpin_umem(struct file *file, unsigned long arg) { struct file_open_data *data = file->private_data; @@ -671,6 +657,7 @@ static int ioctl_sdma_get_streamid(struct file *file, unsigned long arg) return 0; } + static int ioctl_sdma_get_chn(struct file *file, unsigned long arg) { struct file_open_data *data = file->private_data; @@ -702,6 +689,7 @@ static int ioctl_sdma_get_chn(struct file *file, unsigned long arg) list_node->chn_idx = idx; list_add(&list_node->chn_list, &data->non_share_chn_list); pchannel = pdev->channels + idx; + pchannel->cnt_used++; sdma_add_ida_val(pchannel, data->ida); if (copy_to_user((u32 __user *)(uintptr_t)arg, &idx, sizeof(u32))) { @@ -717,6 +705,7 @@ static int ioctl_sdma_get_chn(struct file *file, unsigned long arg) list_del(&list_node->chn_list); bitmap_set(pdev->channel_map, idx - share_chns, 1); pdev->nr_channel_used--; + pchannel->cnt_used--; sdma_reset_ida_val(pchannel); unlock: spin_unlock(&pdev->channel_lock); @@ -756,6 +745,7 @@ static int ioctl_sdma_put_chn(struct file *file, unsigned long arg) if (c->chn_idx == idx) { bitmap_set(pdev->channel_map, idx - share_chns, 1); pdev->nr_channel_used--; + pchannel->cnt_used--; sdma_reset_ida_val(pchannel); dev_dbg(dev, "SDMA put chn %u\n", idx); list_del(&c->chn_list); @@ -966,46 +956,47 @@ static int ioctl_sdma_add_authority_ht(struct file *file, unsigned long arg) free_list: kfree(pid_list); + return ret; } static int sdma_verify_src_dst(struct file_open_data *data, struct pasid_info *pasid, - struct hisi_sdma_sqe_task task_list) + struct hisi_sdma_sqe_task *task_list) { struct device *dev = &data->psdma_dev->pdev->dev; u32 pid = (u32)current->tgid; int ret = -EPERM; - if (task_list.opcode == HISI_SDMA_HBM_CACHE_PRELOAD_MODE) { + if (task_list->opcode == HISI_SDMA_HBM_CACHE_PRELOAD_MODE) { pasid->src_pasid = data->pasid; pasid->dst_pasid = 0; dev_dbg(dev, "Under hbm cache preload mode\n"); return 0; } - if (pid == task_list.src_process_id) { + if (pid == task_list->src_process_id) { pasid->src_pasid = data->pasid; - ret = sdma_check_authority(pasid->src_pasid, task_list.dst_process_id, + ret = sdma_check_authority(pasid->src_pasid, task_list->dst_process_id, current->tgid, &pasid->dst_pasid); - } else if (pid == task_list.dst_process_id) { + } else if (pid == task_list->dst_process_id) { pasid->dst_pasid = data->pasid; - ret = sdma_check_authority(pasid->dst_pasid, task_list.src_process_id, + ret = sdma_check_authority(pasid->dst_pasid, task_list->src_process_id, current->tgid, &pasid->src_pasid); } if (ret < 0) dev_err(dev, "No authority:tgid[%u] src_pid[%u] dst_pid[%u]\n", - pid, task_list.src_process_id, task_list.dst_process_id); + pid, task_list->src_process_id, task_list->dst_process_id); return ret; } static void sdma_fill_sqe(struct hisi_sdma_sq_entry *sq_entry, struct hisi_sdma_sqe_task *task, - struct pasid_info pasid, u32 sq_tail, u32 streamid) + struct pasid_info *pasid, u32 sq_tail, u32 streamid) { sq_entry->opcode = task->opcode; - sq_entry->src_streamid = streamid; - sq_entry->dst_streamid = streamid; + sq_entry->src_streamid = (u16)streamid; + sq_entry->dst_streamid = (u16)streamid; sq_entry->src_addr_l = (u32)(task->src_addr & 0xffffffff); sq_entry->src_addr_h = (u32)(task->src_addr >> HISI_SDMA_LOW_ADDR_SHIFT); sq_entry->dst_addr_l = (u32)(task->dst_addr & 0xffffffff); @@ -1017,9 +1008,9 @@ static void sdma_fill_sqe(struct hisi_sdma_sq_entry *sq_entry, struct hisi_sdma_ sq_entry->mpamns = 1; sq_entry->sssv = 1; sq_entry->dssv = 1; - sq_entry->src_substreamid = pasid.src_pasid; - sq_entry->dst_substreamid = pasid.dst_pasid; - sq_entry->sqe_id = sq_tail; + sq_entry->src_substreamid = (u16)(pasid->src_pasid); + sq_entry->dst_substreamid = (u16)(pasid->dst_pasid); + sq_entry->sqe_id = (u16)sq_tail; sq_entry->src_stride_len = task->src_stride_len; sq_entry->dst_stride_len = task->dst_stride_len; sq_entry->stride_num = task->stride_num; @@ -1080,18 +1071,18 @@ static int sdma_send_task_kernel(struct file_open_data *data, continue; } } - ret = sdma_verify_src_dst(data, &pasid, task_list[i]); + ret = sdma_verify_src_dst(data, &pasid, &task_list[i]); if (ret < 0) { spin_unlock(&pchannel->owner_chn_lock); dev_err(&pdev->pdev->dev, "No correct pid\n"); return ret; } sqe = pchannel->sq_base + sq_tail; - sdma_fill_sqe(sqe, &task_list[i], pasid, sq_tail, pdev->streamid); + sdma_fill_sqe(sqe, &task_list[i], &pasid, sq_tail, pdev->streamid); sq_tail = (sq_tail + 1) & (HISI_SDMA_SQ_LENGTH - 1); } sdma_channel_set_sq_tail(pchannel, sq_tail); - pchannel->sync_info_base->sq_tail = sq_tail; + pchannel->sync_info_base->sq_tail = (u16)sq_tail; spin_unlock(&pchannel->owner_chn_lock); return 0; @@ -1175,6 +1166,7 @@ static int ioctl_sdma_send_task(struct file *file, unsigned long arg) free_list: kfree(task_list); + return ret; } @@ -1464,6 +1456,7 @@ static int sdma_dev_release(struct inode *inode SDMA_UNUSED, struct file *file) list_for_each_entry_safe(c, n, &data->non_share_chn_list, chn_list) { dev_dbg(dev, "Release non_share_chn%u\n", c->chn_idx); pchannel = pdev->channels + c->chn_idx; + pchannel->cnt_used--; sdma_reset_ida_val(pchannel); bitmap_set(pdev->channel_map, c->chn_idx - share_chns, 1); list_del(&c->chn_list); @@ -1496,39 +1489,36 @@ static int sdma_dev_release(struct inode *inode SDMA_UNUSED, struct file *file) } spin_unlock(&pdev->channel_lock); - sdma_put_pause_mmu_notifier(); + /* mm could be flushed by exec/fork, we don't put mmu_notifier at this time */ + if (current->mm == data->cur_file_mm) + sdma_put_pause_mmu_notifier(pdev); + if (data->handle) iommu_sva_unbind_device(data->handle); - sdma_put_resume_mmu_notifier(); + if (current->mm == data->cur_file_mm) - atomic_sub(1, &ttl_processes); + sdma_put_resume_mmu_notifier(pdev); sdma_hash_free_entry(data->ida); - sdma_del_pid_ref(pdev, pid); + sdma_del_pid_ref(pid); ida_free(g_info.fd_ida, data->ida); kfree(file->private_data); file->private_data = NULL; + return 0; } static int remap_addr_range(u32 chn_num, u64 offset, u64 size, struct hisi_sdma_channel *chn_base, int ida) { + u64 reg_max_size = max_t(u64, HISI_SDMA_REG_SIZE, PAGE_SIZE); + u64 sq_max_size = max_t(u64, HISI_SDMA_SQ_SIZE, PAGE_SIZE); + u64 cq_max_size = max_t(u64, HISI_SDMA_CQ_SIZE, PAGE_SIZE); struct hisi_sdma_channel *pchannel; bool mode = *(g_info.sdma_mode); - u64 reg_max_size = PAGE_SIZE; - u64 sq_max_size = PAGE_SIZE; - u64 cq_max_size = PAGE_SIZE; u64 sync_size; - if (HISI_SDMA_REG_SIZE >= PAGE_SIZE) - reg_max_size = HISI_SDMA_REG_SIZE; - if (HISI_SDMA_SQ_SIZE >= PAGE_SIZE) - sq_max_size = HISI_SDMA_SQ_SIZE; - if (HISI_SDMA_CQ_SIZE >= PAGE_SIZE) - cq_max_size = HISI_SDMA_CQ_SIZE; - sync_size = (u64)((sizeof(struct hisi_sdma_queue_info) + PAGE_SIZE - ALIGN_NUM) / PAGE_SIZE * PAGE_SIZE); @@ -1638,32 +1628,30 @@ static int sdma_dev_mmap(struct file *file, struct vm_area_struct *vma) return ret; } -static const struct file_operations sdma_core_fops = { - .owner = THIS_MODULE, - .open = sdma_core_open, - .read = sdma_read_info, - .release = sdma_dev_release, - .unlocked_ioctl = sdma_dev_ioctl, - .mmap = sdma_dev_mmap, +static const struct file_operations sdma_cdev_core_fops = { + .owner = THIS_MODULE, + .open = sdma_core_open, + .read = sdma_read_info, + .release = sdma_dev_release, + .unlocked_ioctl = sdma_dev_ioctl, + .mmap = sdma_dev_mmap, }; void sdma_cdev_init(struct cdev *cdev) { - cdev_init(cdev, &sdma_core_fops); + cdev_init(cdev, &sdma_cdev_core_fops); cdev->owner = THIS_MODULE; } void sdma_info_sync_cdev(struct hisi_sdma_core_device *p, u32 *share_chns, struct ida *fd_ida, - bool *safe_mode, struct mutex *mutex_lock) -{ - g_info.core_dev = p; - g_info.fd_ida = fd_ida; - g_info.share_chns = share_chns; - g_info.sdma_mode = safe_mode; - g_info.mutex_lock = mutex_lock; - INIT_LIST_HEAD(&g_info.sdma_pause_mm_list); - INIT_LIST_HEAD(&g_info.sdma_resume_mm_list); - - atomic_set(&ttl_processes, 0); - atomic_set(&exit_processes, 0); + bool *safe_mode) +{ + struct hisi_sdma_global_info *sdma_global_info = &g_info; + + sdma_global_info->core_dev = p; + sdma_global_info->fd_ida = fd_ida; + sdma_global_info->share_chns = share_chns; + sdma_global_info->sdma_mode = safe_mode; + + hash_init(sdma_global_info->sdma_pid_ref_ht); } diff --git a/drivers/misc/sdma-dae/sdma_dbg.c b/drivers/misc/sdma-dae/sdma_dbg.c index dbd361f09a14..17097e84a892 100644 --- a/drivers/misc/sdma-dae/sdma_dbg.c +++ b/drivers/misc/sdma-dae/sdma_dbg.c @@ -11,7 +11,7 @@ #define SINGLE_CHANNEL_SELECTED 1 #define R_R_R 0444 -static struct hisi_sdma_global_info dbg_g_info; +static struct hisi_sdma_debugfs_info dbg_g_info; static u32 debug_mode; static u32 device_id; static u32 channel_id; diff --git a/drivers/misc/sdma-dae/sdma_hal.h b/drivers/misc/sdma-dae/sdma_hal.h index 76c525a630d0..437fb755bca3 100644 --- a/drivers/misc/sdma-dae/sdma_hal.h +++ b/drivers/misc/sdma-dae/sdma_hal.h @@ -4,7 +4,6 @@ #include #include -#include #include #include #include @@ -21,24 +20,8 @@ #define HISI_SDMA_HAL_HASH_BUCKETS_BITS 8 /* HISI_SDMA_POLL_TIMOUT_VAL */ -#define SDMA_POLL_ERR_TIMEOUT 110 #define SDMA_POLL_DELAY 1 -#define SDMA_POLL_TIMEOUT 80 - -#define HISI_SDMA_IO_READ32_POLL_TIMEOUT(_addr, _val, _cond, _delay_us, _timeout_us) \ - ({ \ - uint32_t __timeout = 0; \ - uint32_t __delay = (_delay_us); \ - while (__timeout < (_timeout_us)) { \ - (_val) = readl(_addr); \ - if (_cond) \ - break; \ - __timeout += (__delay); \ - mdelay(__delay); \ - } \ - (_val) = readl(_addr); \ - (_cond) ? 0 : -SDMA_POLL_ERR_TIMEOUT; \ - }) +#define SDMA_POLL_TIMEOUT 100 /** * struct hisi_sdma_channel - Information about one channel in the SDMA device @@ -72,6 +55,12 @@ struct hisi_sdma_channel_node { struct hlist_node node; }; +/** + * struct hisi_sdma_pid_ref_hte - SDMA device pid table entry + * @pid: Process's id + * @ref: Number of devices opened in this process + * @node: Entry node of process in the hash table + */ struct hisi_sdma_pid_ref_hte { u32 pid; u32 ref; @@ -89,6 +78,11 @@ struct hisi_sdma_pid_ref_hte { * @io_base: io_orig_base + 32 channel address offsets * @base_addr: SDMA I/O base phyisical address * @name: SDMA device name in the /dev directory + * @sdma_dev_ref_cnt: SDMA device register mmu_notifier count + * @dev_exit_ref_cnt: SDMA device trigger mmu_notifier count + * @dev_exit_flag: SDMA device pause channels flag (1 for paused) + * @sdma_pause_mm_list: List of SDMA devices which register pause mmu_notifier + * @sdma_resume_mm_list: List of SDMA devices which register resume mmu_notifier */ struct hisi_sdma_device { u16 idx; @@ -114,8 +108,14 @@ struct hisi_sdma_device { int irq_cnt; int irq[SDMA_IRQ_NUM_MAX]; - DECLARE_HASHTABLE(sdma_pid_ref_ht, HISI_SDMA_HAL_HASH_BUCKETS_BITS); - spinlock_t pid_lock; + + atomic_t sdma_dev_ref_cnt; + atomic_t dev_exit_ref_cnt; + atomic_t dev_exit_flag; + + struct mutex mutex_lock; + struct list_head sdma_pause_mm_list; + struct list_head sdma_resume_mm_list; }; struct hisi_sdma_core_device { @@ -130,18 +130,20 @@ struct hisi_sdma_global_info { bool *sdma_mode; struct hisi_sdma_core_device *core_dev; struct ida *fd_ida; - struct mutex *mutex_lock; - struct list_head sdma_pause_mm_list; - struct list_head sdma_resume_mm_list; + DECLARE_HASHTABLE(sdma_pid_ref_ht, HISI_SDMA_HAL_HASH_BUCKETS_BITS); }; -void sdma_channel_reset_sq_cq(struct hisi_sdma_channel *pchan); -void sdma_clear_pid_ref(struct hisi_sdma_device *psdma_dev); +struct hisi_sdma_debugfs_info { + u32 *share_chns; + struct hisi_sdma_core_device *core_dev; +}; + +void sdma_cdev_init(struct cdev *cdev); +void sdma_clear_pid_ref(void); void sdma_clear_ida_ref(struct hisi_sdma_channel *pchannel); int sdma_create_dbg_node(struct dentry *sdma_dbgfs_dir); -void sdma_cdev_init(struct cdev *cdev); void sdma_info_sync_cdev(struct hisi_sdma_core_device *p, u32 *share_chns, struct ida *fd_ida, - bool *safe_mode, struct mutex *mutex_lock); + bool *safe_mode); void sdma_info_sync_dbg(struct hisi_sdma_core_device *p, u32 *share_chns); static inline void chn_set_val(struct hisi_sdma_channel *pchan, int reg, u32 val, u32 mask) @@ -151,7 +153,6 @@ static inline void chn_set_val(struct hisi_sdma_channel *pchan, int reg, u32 val reg_val &= ~mask; reg_val |= FIELD_PREP(mask, val); /* calculate reg_val before writing into register */ - wmb(); writel(reg_val, pchan->io_base + reg); } @@ -165,7 +166,12 @@ static inline u32 chn_get_val(struct hisi_sdma_channel *pchan, int reg, u32 mask static inline void sdma_channel_set_pause(struct hisi_sdma_channel *pchan) { - chn_set_val(pchan, HISI_SDMA_CH_TEST_REG, 1, HISI_SDMA_CH_PAUSE_MSK); + u32 reg_val = readl(pchan->io_base + HISI_SDMA_CH_TEST_REG); + + reg_val &= ~HISI_SDMA_CH_PAUSE_MSK; + reg_val |= FIELD_PREP(HISI_SDMA_CH_PAUSE_MSK, 1); + + writel(reg_val, pchan->io_base + HISI_SDMA_CH_TEST_REG); } static inline bool sdma_channel_is_paused(struct hisi_sdma_channel *pchan) @@ -208,7 +214,6 @@ static inline void sdma_channel_write_resume(struct hisi_sdma_channel *pchan) reg_val &= ~HISI_SDMA_CH_RESUME_MSK; reg_val |= FIELD_PREP(HISI_SDMA_CH_RESUME_MSK, 1); - wmb(); writel(reg_val, pchan->io_base + HISI_SDMA_CH_TEST_REG); } @@ -262,7 +267,12 @@ static inline u32 sdma_channel_get_sq_head(struct hisi_sdma_channel *pchan) static inline void sdma_channel_set_cq_head(struct hisi_sdma_channel *pchan, u32 val) { - chn_set_val(pchan, HISI_SDMA_CH_CQHDBR_REG, val, HISI_SDMA_U32_MSK); + u32 reg_val = readl(pchan->io_base + HISI_SDMA_CH_CQHDBR_REG); + + reg_val &= ~HISI_SDMA_U32_MSK; + reg_val |= FIELD_PREP(HISI_SDMA_U32_MSK, 1); + + writel(reg_val, pchan->io_base + HISI_SDMA_CH_CQHDBR_REG); } static inline u32 sdma_channel_get_cq_tail(struct hisi_sdma_channel *pchan) @@ -287,9 +297,6 @@ static inline u32 sdma_channel_get_err_status(struct hisi_sdma_channel *pchan) static inline void sdma_channel_clear_ioe_status(void __iomem *io_addr) { - union sdmam_irq_status reg_val = {0}; - - reg_val.bits.ch_ioe_status = 1; writel(HISI_SDMA_U32_MSK, io_addr + HISI_SDMA_IRQ_STATUS); } diff --git a/drivers/misc/sdma-dae/sdma_irq.c b/drivers/misc/sdma-dae/sdma_irq.c index ad4c588758fc..c8a3b13df05d 100644 --- a/drivers/misc/sdma-dae/sdma_irq.c +++ b/drivers/misc/sdma-dae/sdma_irq.c @@ -99,12 +99,10 @@ int sdma_irq_init(struct hisi_sdma_device *sdma) void sdma_irq_deinit(struct hisi_sdma_device *sdma) { - struct platform_device *pdev; void __iomem *io_addr; u32 i; - pdev = sdma->pdev; - for (i = 0; i < sdma->nr_channel + HISI_STARS_CHN_NUM; i++) { + for (i = HISI_STARS_CHN_NUM; i < sdma->nr_channel + HISI_STARS_CHN_NUM; i++) { io_addr = sdma->io_orig_base + i * HISI_SDMA_CHANNEL_IOMEM_SIZE; sdma_channel_set_irq_mask(io_addr, SDMA_IOC_IOE_MASKED_STATUS); } diff --git a/drivers/misc/sdma-dae/sdma_main.c b/drivers/misc/sdma-dae/sdma_main.c index 81558862ded8..2e01ba5715e4 100644 --- a/drivers/misc/sdma-dae/sdma_main.c +++ b/drivers/misc/sdma-dae/sdma_main.c @@ -19,7 +19,7 @@ #define BASE_DIR "sdma" /* Subdir in /sys/kernel/debug/ */ #define UPPER_SHIFT 32 #define MAX_INPUT_LENGTH 128 -#define SDMA_VERSION "1.0.4" +#define SDMA_VERSION "1.0.5" u32 share_chns = 16; module_param(share_chns, uint, RW_R_R); @@ -30,7 +30,6 @@ module_param(safe_mode, bool, RW_R_R); MODULE_PARM_DESC(safe_mode, "| 0 - fast_mode| 1 - safe_mode(default)|"); struct ida fd_ida; -struct mutex g_mutex_lock; struct hisi_sdma_core_device hisi_sdma_core_device = {0}; static struct class *sdma_class; static struct dentry *sdma_dbgfs_dir; @@ -90,7 +89,7 @@ static void sdma_channel_init(struct hisi_sdma_channel *pchan) sdma_channel_enable(pchan); } -void sdma_channel_reset_sq_cq(struct hisi_sdma_channel *pchan) +static void sdma_channel_reset_sq_cq(struct hisi_sdma_channel *pchan) { u32 cq_head, cq_tail; @@ -108,6 +107,7 @@ void sdma_channel_reset_sq_cq(struct hisi_sdma_channel *pchan) static void sdma_channel_reset(struct hisi_sdma_channel *pchan) { + u32 cq_tail; int i = 0; sdma_channel_reset_sq_cq(pchan); @@ -121,6 +121,8 @@ static void sdma_channel_reset(struct hisi_sdma_channel *pchan) } i = 0; while (!sdma_channel_is_quiescent(pchan)) { + cq_tail = sdma_channel_get_cq_tail(pchan); + sdma_channel_set_cq_head(pchan, cq_tail); msleep(HISI_SDMA_FSM_INTERVAL); if (++i > HISI_SDMA_FSM_TIMEOUT) { pr_warn("Chn%u cannot get quiescent\n", pchan->idx); @@ -175,9 +177,9 @@ static void sdma_destroy_channels(struct hisi_sdma_device *psdma_dev) static int sdma_init_channels(struct hisi_sdma_device *psdma_dev) { - u32 chn_num = psdma_dev->nr_channel; + u16 chn_num = psdma_dev->nr_channel; struct hisi_sdma_channel *pchan; - u32 i; + u16 i; psdma_dev->channels = kcalloc_node(chn_num, sizeof(struct hisi_sdma_channel), GFP_KERNEL, psdma_dev->node_idx); @@ -194,11 +196,22 @@ static int sdma_init_channels(struct hisi_sdma_device *psdma_dev) pchan->ida = -1; spin_lock_init(&pchan->owner_chn_lock); - if (sdma_channel_alloc_sq_cq(pchan, psdma_dev->node_idx) == false) + if (!sdma_channel_alloc_sq_cq(pchan, psdma_dev->node_idx)) goto err_out; pchan->io_base = psdma_dev->io_base + i * HISI_SDMA_CHANNEL_IOMEM_SIZE; + if (sdma_channel_is_paused(pchan) && sdma_channel_is_quiescent(pchan)) { + sdma_channel_write_resume(pchan); + continue; + } + + if (sdma_channel_is_paused(pchan) && !sdma_channel_is_quiescent(pchan)) { + sdma_channel_reset_sq_cq(pchan); + pr_warn("Chn%u not quiescent, not init\n", pchan->idx); + continue; + } + sdma_channel_disable(pchan); sdma_channel_init(pchan); } @@ -218,9 +231,8 @@ static int sdma_init_channels(struct hisi_sdma_device *psdma_dev) static int sdma_device_add(struct hisi_sdma_device *psdma_dev) { - u32 idx = psdma_dev->idx; + u16 idx = psdma_dev->idx; struct cdev *cdev = NULL; - u32 sdma_minor; int devno; int ret; @@ -230,10 +242,9 @@ static int sdma_device_add(struct hisi_sdma_device *psdma_dev) } spin_lock(&hisi_sdma_core_device.device_lock); - sdma_minor = idx; hisi_sdma_core_device.sdma_devices[idx] = psdma_dev; cdev = &hisi_sdma_core_device.sdma_devices[idx]->cdev; - devno = MKDEV(hisi_sdma_core_device.sdma_major, sdma_minor); + devno = MKDEV(hisi_sdma_core_device.sdma_major, idx); sdma_cdev_init(cdev); ret = cdev_add(cdev, devno, 1); @@ -315,15 +326,15 @@ static int parse_sdma(struct hisi_sdma_device *psdma_dev, struct platform_device if (device_property_read_u32(&pdev->dev, "sdma-chn-num", &nr_channel)) { pr_err("ACPI sdma-chn-num get failed!\n"); return -EINVAL; - } else { - if (nr_channel <= HISI_STARS_CHN_NUM || nr_channel > - HISI_SDMA_DEFAULT_CHANNEL_NUM + HISI_STARS_CHN_NUM) { - pr_err("ACPI sdma-chn-num = %u not as required\n", nr_channel); - return -EINVAL; - } - psdma_dev->nr_channel = (u16)(nr_channel - HISI_STARS_CHN_NUM); } + if (nr_channel <= HISI_STARS_CHN_NUM || nr_channel > HISI_SDMA_MAX_CHN_NUM) { + pr_err("ACPI sdma-chn-num = %u not as required\n", nr_channel); + return -EINVAL; + } + + psdma_dev->nr_channel = (u16)(nr_channel - HISI_STARS_CHN_NUM); + return 0; } @@ -435,19 +446,24 @@ static int sdma_device_probe(struct platform_device *pdev) psdma_dev->streamid = pdev->dev.iommu->fwspec->ids[0]; spin_lock_init(&psdma_dev->channel_lock); - hash_init(psdma_dev->sdma_pid_ref_ht); - spin_lock_init(&psdma_dev->pid_lock); ret = sdma_device_add(psdma_dev); if (ret) goto sva_device_shutdown; + atomic_set(&psdma_dev->sdma_dev_ref_cnt, 0); + atomic_set(&psdma_dev->dev_exit_ref_cnt, 0); + atomic_set(&psdma_dev->dev_exit_flag, 0); + + INIT_LIST_HEAD(&psdma_dev->sdma_pause_mm_list); + INIT_LIST_HEAD(&psdma_dev->sdma_resume_mm_list); + mutex_init(&psdma_dev->mutex_lock); + dev_info(&pdev->dev, "SDMA%u registered\n", psdma_dev->idx); return 0; sva_device_shutdown: - sdma_clear_pid_ref(psdma_dev); iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); deinit_device: @@ -462,8 +478,8 @@ static int sdma_device_remove(struct platform_device *pdev) { struct hisi_sdma_device *psdma_dev = dev_get_drvdata(&pdev->dev); + mutex_destroy(&psdma_dev->mutex_lock); sdma_device_delete(psdma_dev); - sdma_clear_pid_ref(psdma_dev); iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); sdma_deinit_device_info(psdma_dev); @@ -526,8 +542,7 @@ static int __init sdma_driver_init(void) long ret; ida_init(&fd_ida); - sdma_info_sync_cdev(&hisi_sdma_core_device, &share_chns, &fd_ida, &safe_mode, - &g_mutex_lock); + sdma_info_sync_cdev(&hisi_sdma_core_device, &share_chns, &fd_ida, &safe_mode); sdma_info_sync_dbg(&hisi_sdma_core_device, &share_chns); sdma_class = class_create(THIS_MODULE, "sdma"); @@ -563,8 +578,6 @@ static int __init sdma_driver_init(void) goto umem_hash_free; } - mutex_init(&g_mutex_lock); - return 0; umem_hash_free: @@ -585,9 +598,9 @@ static int __init sdma_driver_init(void) static void __exit sdma_driver_exit(void) { - mutex_destroy(&g_mutex_lock); sdma_authority_ht_free(); sdma_hash_free(); + sdma_clear_pid_ref(); platform_driver_unregister(&sdma_driver); debugfs_remove_recursive(sdma_dbgfs_dir); diff --git a/drivers/misc/sdma-dae/sdma_umem.c b/drivers/misc/sdma-dae/sdma_umem.c index e63fc19d5713..55d687160af2 100644 --- a/drivers/misc/sdma-dae/sdma_umem.c +++ b/drivers/misc/sdma-dae/sdma_umem.c @@ -22,8 +22,7 @@ static void free_region(struct list_head *list_head) list_for_each_entry_safe(node_remove, node_next, list_head, list) { unpin_user_pages(node_remove->pnode.page_list, node_remove->pnode.pinned); - free_pages((uintptr_t)(void *)node_remove->pnode.page_list, - get_order(node_remove->pnode.pinned * sizeof(struct page *))); + free_page((uintptr_t)(void *)node_remove->pnode.page_list); list_del(&node_remove->list); kfree(node_remove); } @@ -94,7 +93,7 @@ void sdma_hash_free(void) g_hash_table = NULL; } -static int record_umem(u64 addr, struct list_head *list_head, int ida, u64 *cookie) +static int sdma_record_umem(u64 addr, struct list_head *list_head, int ida, u64 *cookie) { struct hash_entry *entry; bool entry_find = true; @@ -139,7 +138,7 @@ static int record_umem(u64 addr, struct list_head *list_head, int ida, u64 *cook pmem->idr = idr; spin_unlock(&g_hash_table->hash_lock); - *cookie = ((u64)ida << COOKIE_IDA_SHIFT) + idr; + *cookie = ((u64)ida << COOKIE_IDA_SHIFT) | ((u64)idr & COOKIE_IDA_MASK); pr_debug("Record addr: ida = %d, idr = %d\n", ida, idr); return 0; @@ -148,32 +147,40 @@ static int record_umem(u64 addr, struct list_head *list_head, int ida, u64 *cook kfree(entry); free_pmem: kfree(pmem); + return ret; } -static int pin_umem(u64 addr, int npages, struct list_head *p_head) +/** + * sdma_pin_umem - SDMA kernel driver pin memory of userspace. + * + * @addr: memory virtual address be pinned + * @npages: number of pages be pinned + * @p_head: pointer points towards pin memory + */ +static int sdma_pin_umem(u64 addr, u32 npages, struct list_head *p_head) { - int pinned, to_pin_pages, unpin_pages, ret = 0; - size_t node_size = sizeof(struct p_list); + u64 pin_addr = addr & PAGE_MASK; + int pinned, to_pin_pages; struct page **page_list; struct p_list *cur_node; - u64 pin_addr = addr; - - to_pin_pages = unpin_pages = npages; - while (unpin_pages != 0) { - if (to_pin_pages > HISI_SDMA_MAX_ALLOC_SIZE / sizeof(struct page *)) - to_pin_pages = HISI_SDMA_MAX_ALLOC_SIZE / sizeof(struct page *); - page_list = (struct page **)(uintptr_t)__get_free_pages(GFP_KERNEL, - get_order(to_pin_pages * sizeof(struct page *))); + int ret = 0; + + while (npages) { + /* page_list mostly record (PAGE_SIZE / 8) pages */ + page_list = (struct page **)__get_free_page(GFP_KERNEL); if (!page_list) { pr_err("SDMA failed to alloc page list!\n"); - return -ENOMEM; + ret = -ENOMEM; + goto exit; } + to_pin_pages = min_t(unsigned int, npages, PAGE_SIZE / sizeof(struct page *)); + pinned = pin_user_pages_fast(pin_addr, to_pin_pages, FOLL_WRITE, page_list); if (pinned < 0) { - pr_err("SDMA failed to pin user pages!\n"); ret = pinned; + pr_err("SDMA failed to pin user pages!\n"); goto free_pages; } else if (pinned != to_pin_pages) { pr_err("Invalid number of pages. SDMA pinned %d pages, expect %d pages\n", @@ -181,9 +188,9 @@ static int pin_umem(u64 addr, int npages, struct list_head *p_head) ret = -EINVAL; goto unpin_page; } + npages -= pinned; - cur_node = NULL; - cur_node = kzalloc(node_size, GFP_KERNEL); + cur_node = kzalloc(sizeof(struct p_list), GFP_KERNEL); if (!cur_node) { ret = -ENOMEM; goto unpin_page; @@ -191,16 +198,15 @@ static int pin_umem(u64 addr, int npages, struct list_head *p_head) cur_node->pnode.page_list = page_list; cur_node->pnode.pinned = pinned; list_add(&cur_node->list, p_head); - unpin_pages -= to_pin_pages; - if (unpin_pages > 0) - pin_addr += to_pin_pages * PAGE_SIZE; - to_pin_pages = unpin_pages; + pin_addr += (u64)pinned * PAGE_SIZE; } - goto exit; + + return 0; + unpin_page: unpin_user_pages(page_list, pinned); free_pages: - free_pages((uintptr_t)(void *)page_list, get_order(to_pin_pages * sizeof(struct page *))); + free_page((uintptr_t)(void *)page_list); exit: return ret; } @@ -208,7 +214,7 @@ static int pin_umem(u64 addr, int npages, struct list_head *p_head) int sdma_umem_get(u64 addr, u32 size, int ida, u64 *cookie) { struct list_head *p_head; - int npages; + u32 npages; int ret; /* Check overflow */ @@ -223,7 +229,7 @@ int sdma_umem_get(u64 addr, u32 size, int ida, u64 *cookie) INIT_LIST_HEAD(p_head); npages = (PAGE_ALIGN(addr + size) - ALIGN_DOWN(addr, PAGE_SIZE)) / PAGE_SIZE; - ret = pin_umem(addr, npages, p_head); + ret = sdma_pin_umem(addr, npages, p_head); if (ret != 0) { pr_err("SDMA failed to pin_umem\n"); free_region(p_head); @@ -231,7 +237,7 @@ int sdma_umem_get(u64 addr, u32 size, int ida, u64 *cookie) return ret; } - ret = record_umem(addr, p_head, ida, cookie); + ret = sdma_record_umem(addr, p_head, ida, cookie); if (ret) { pr_err("SDMA failed to record umem\n"); free_region(p_head); @@ -272,5 +278,6 @@ int sdma_umem_release(u64 cookie) free_region(pmem->list_head); kfree((void *)pmem->list_head); kfree(pmem); + return 0; } diff --git a/drivers/misc/sdma-dae/sdma_umem.h b/drivers/misc/sdma-dae/sdma_umem.h index 23573f653a01..8c6755ce7c81 100644 --- a/drivers/misc/sdma-dae/sdma_umem.h +++ b/drivers/misc/sdma-dae/sdma_umem.h @@ -5,7 +5,8 @@ #include #include #include -#include "sdma_hal.h" +#include +#include #define HISI_SDMA_HASH_BUCKETS_BITS 3 #define COOKIE_IDA_SHIFT 32 -- Gitee