Lines Matching refs:pcl
117 static inline bool z_erofs_is_inline_pcluster(struct z_erofs_pcluster *pcl) in z_erofs_is_inline_pcluster() argument
119 return !pcl->obj.index; in z_erofs_is_inline_pcluster()
122 static inline unsigned int z_erofs_pclusterpages(struct z_erofs_pcluster *pcl) in z_erofs_pclusterpages() argument
124 if (z_erofs_is_inline_pcluster(pcl)) in z_erofs_pclusterpages()
126 return pcl->pclusterpages; in z_erofs_pclusterpages()
312 struct z_erofs_pcluster *pcl; in z_erofs_alloc_pcluster() local
317 pcl = kmem_cache_zalloc(pcs->slab, GFP_NOFS); in z_erofs_alloc_pcluster()
318 if (!pcl) in z_erofs_alloc_pcluster()
320 pcl->pclusterpages = nrpages; in z_erofs_alloc_pcluster()
321 return pcl; in z_erofs_alloc_pcluster()
326 static void z_erofs_free_pcluster(struct z_erofs_pcluster *pcl) in z_erofs_free_pcluster() argument
328 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); in z_erofs_free_pcluster()
337 kmem_cache_free(pcs->slab, pcl); in z_erofs_free_pcluster()
549 struct z_erofs_pcluster *pcl, *tailpcl; member
587 struct z_erofs_pcluster *pcl = fe->pcl; in z_erofs_bind_cache() local
601 for (i = 0; i < pcl->pclusterpages; ++i) { in z_erofs_bind_cache()
607 if (READ_ONCE(pcl->compressed_bvecs[i].page)) in z_erofs_bind_cache()
610 page = find_get_page(mc, pcl->obj.index + i); in z_erofs_bind_cache()
632 if (!cmpxchg_relaxed(&pcl->compressed_bvecs[i].page, NULL, t)) in z_erofs_bind_cache()
653 struct z_erofs_pcluster *const pcl = in erofs_try_to_free_all_cached_pages() local
657 DBG_BUGON(z_erofs_is_inline_pcluster(pcl)); in erofs_try_to_free_all_cached_pages()
662 for (i = 0; i < pcl->pclusterpages; ++i) { in erofs_try_to_free_all_cached_pages()
663 struct page *page = pcl->compressed_bvecs[i].page; in erofs_try_to_free_all_cached_pages()
676 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL); in erofs_try_to_free_all_cached_pages()
685 struct z_erofs_pcluster *const pcl = (void *)page_private(page); in erofs_try_to_free_cached_page() local
688 if (!erofs_workgroup_try_to_freeze(&pcl->obj, 1)) in erofs_try_to_free_cached_page()
692 DBG_BUGON(z_erofs_is_inline_pcluster(pcl)); in erofs_try_to_free_cached_page()
693 for (i = 0; i < pcl->pclusterpages; ++i) { in erofs_try_to_free_cached_page()
694 if (pcl->compressed_bvecs[i].page == page) { in erofs_try_to_free_cached_page()
695 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL); in erofs_try_to_free_cached_page()
700 erofs_workgroup_unfreeze(&pcl->obj, 1); in erofs_try_to_free_cached_page()
709 struct z_erofs_pcluster *const pcl = fe->pcl; in z_erofs_try_inplace_io() local
712 if (!cmpxchg(&pcl->compressed_bvecs[--fe->icur].page, in z_erofs_try_inplace_io()
714 pcl->compressed_bvecs[fe->icur] = *bvec; in z_erofs_try_inplace_io()
737 fe->pcl->vcnt += (ret >= 0); in z_erofs_attach_page()
743 struct z_erofs_pcluster *pcl = f->pcl; in z_erofs_try_to_claim_pcluster() local
747 if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_NIL, in z_erofs_try_to_claim_pcluster()
749 *owned_head = &pcl->next; in z_erofs_try_to_claim_pcluster()
759 if (*owned_head != &pcl->next && pcl != f->tailpcl && in z_erofs_try_to_claim_pcluster()
760 cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL, in z_erofs_try_to_claim_pcluster()
775 struct z_erofs_pcluster *pcl; in z_erofs_register_pcluster() local
786 pcl = z_erofs_alloc_pcluster(ztailpacking ? 1 : in z_erofs_register_pcluster()
788 if (IS_ERR(pcl)) in z_erofs_register_pcluster()
789 return PTR_ERR(pcl); in z_erofs_register_pcluster()
791 atomic_set(&pcl->obj.refcount, 1); in z_erofs_register_pcluster()
792 pcl->algorithmformat = map->m_algorithmformat; in z_erofs_register_pcluster()
793 pcl->length = 0; in z_erofs_register_pcluster()
794 pcl->partial = true; in z_erofs_register_pcluster()
797 pcl->next = fe->owned_head; in z_erofs_register_pcluster()
798 pcl->pageofs_out = map->m_la & ~PAGE_MASK; in z_erofs_register_pcluster()
805 mutex_init(&pcl->lock); in z_erofs_register_pcluster()
806 DBG_BUGON(!mutex_trylock(&pcl->lock)); in z_erofs_register_pcluster()
809 pcl->obj.index = 0; /* which indicates ztailpacking */ in z_erofs_register_pcluster()
810 pcl->pageofs_in = erofs_blkoff(map->m_pa); in z_erofs_register_pcluster()
811 pcl->tailpacking_size = map->m_plen; in z_erofs_register_pcluster()
813 pcl->obj.index = map->m_pa >> PAGE_SHIFT; in z_erofs_register_pcluster()
815 grp = erofs_insert_workgroup(fe->inode->i_sb, &pcl->obj); in z_erofs_register_pcluster()
821 if (grp != &pcl->obj) { in z_erofs_register_pcluster()
822 fe->pcl = container_of(grp, in z_erofs_register_pcluster()
830 fe->tailpcl = pcl; in z_erofs_register_pcluster()
831 fe->owned_head = &pcl->next; in z_erofs_register_pcluster()
832 fe->pcl = pcl; in z_erofs_register_pcluster()
836 mutex_unlock(&pcl->lock); in z_erofs_register_pcluster()
837 z_erofs_free_pcluster(pcl); in z_erofs_register_pcluster()
847 DBG_BUGON(fe->pcl); in z_erofs_collector_begin()
862 fe->pcl = container_of(grp, struct z_erofs_pcluster, obj); in z_erofs_collector_begin()
869 mutex_lock(&fe->pcl->lock); in z_erofs_collector_begin()
872 fe->tailpcl = fe->pcl; in z_erofs_collector_begin()
878 z_erofs_bvec_iter_begin(&fe->biter, &fe->pcl->bvset, in z_erofs_collector_begin()
879 Z_EROFS_INLINE_BVECS, fe->pcl->vcnt); in z_erofs_collector_begin()
881 fe->icur = z_erofs_pclusterpages(fe->pcl); in z_erofs_collector_begin()
897 struct z_erofs_pcluster *const pcl = in erofs_workgroup_free_rcu() local
900 call_rcu(&pcl->rcu, z_erofs_rcu_callback); in erofs_workgroup_free_rcu()
905 struct z_erofs_pcluster *pcl = fe->pcl; in z_erofs_collector_end() local
907 if (!pcl) in z_erofs_collector_end()
911 mutex_unlock(&pcl->lock); in z_erofs_collector_end()
923 erofs_workgroup_put(&pcl->obj); in z_erofs_collector_end()
925 fe->pcl = NULL; in z_erofs_collector_end()
991 if (fe->pcl) in z_erofs_do_read_page()
1004 if (z_erofs_is_inline_pcluster(fe->pcl)) { in z_erofs_do_read_page()
1016 WRITE_ONCE(fe->pcl->compressed_bvecs[0].page, in z_erofs_do_read_page()
1083 if (fe->pcl->pageofs_out != (map->m_la & ~PAGE_MASK)) in z_erofs_do_read_page()
1084 fe->pcl->multibases = true; in z_erofs_do_read_page()
1085 if (fe->pcl->length < offset + end - map->m_la) { in z_erofs_do_read_page()
1086 fe->pcl->length = offset + end - map->m_la; in z_erofs_do_read_page()
1087 fe->pcl->pageofs_out = map->m_la & ~PAGE_MASK; in z_erofs_do_read_page()
1091 fe->pcl->length == map->m_llen) in z_erofs_do_read_page()
1092 fe->pcl->partial = false; in z_erofs_do_read_page()
1135 struct z_erofs_pcluster *pcl; member
1157 if (!((bvec->offset + be->pcl->pageofs_out) & ~PAGE_MASK)) { in z_erofs_do_decompressed_bvec()
1160 pgnr = (bvec->offset + be->pcl->pageofs_out) >> PAGE_SHIFT; in z_erofs_do_decompressed_bvec()
1177 unsigned int off0 = be->pcl->pageofs_out; in z_erofs_fill_other_copies()
1187 end = min_t(unsigned int, be->pcl->length - bvi->bvec.offset, in z_erofs_fill_other_copies()
1220 struct z_erofs_pcluster *pcl = be->pcl; in z_erofs_parse_out_bvecs() local
1225 z_erofs_bvec_iter_begin(&biter, &pcl->bvset, Z_EROFS_INLINE_BVECS, 0); in z_erofs_parse_out_bvecs()
1226 for (i = 0; i < pcl->vcnt; ++i) { in z_erofs_parse_out_bvecs()
1246 struct z_erofs_pcluster *pcl = be->pcl; in z_erofs_parse_in_bvecs() local
1247 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); in z_erofs_parse_in_bvecs()
1252 struct z_erofs_bvec *bvec = &pcl->compressed_bvecs[i]; in z_erofs_parse_in_bvecs()
1262 if (z_erofs_is_inline_pcluster(pcl)) { in z_erofs_parse_in_bvecs()
1289 struct z_erofs_pcluster *pcl = be->pcl; in z_erofs_decompress_pcluster() local
1290 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); in z_erofs_decompress_pcluster()
1296 mutex_lock(&pcl->lock); in z_erofs_decompress_pcluster()
1297 be->nr_pages = PAGE_ALIGN(pcl->length + pcl->pageofs_out) >> PAGE_SHIFT; in z_erofs_decompress_pcluster()
1329 if (z_erofs_is_inline_pcluster(pcl)) in z_erofs_decompress_pcluster()
1330 inputsize = pcl->tailpacking_size; in z_erofs_decompress_pcluster()
1338 .pageofs_in = pcl->pageofs_in, in z_erofs_decompress_pcluster()
1339 .pageofs_out = pcl->pageofs_out, in z_erofs_decompress_pcluster()
1341 .outputsize = pcl->length, in z_erofs_decompress_pcluster()
1342 .alg = pcl->algorithmformat, in z_erofs_decompress_pcluster()
1344 .partial_decoding = pcl->partial, in z_erofs_decompress_pcluster()
1345 .fillgaps = pcl->multibases, in z_erofs_decompress_pcluster()
1350 if (z_erofs_is_inline_pcluster(pcl)) { in z_erofs_decompress_pcluster()
1351 page = pcl->compressed_bvecs[0].page; in z_erofs_decompress_pcluster()
1352 WRITE_ONCE(pcl->compressed_bvecs[0].page, NULL); in z_erofs_decompress_pcluster()
1356 page = pcl->compressed_bvecs[i].page; in z_erofs_decompress_pcluster()
1363 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL); in z_erofs_decompress_pcluster()
1389 pcl->length = 0; in z_erofs_decompress_pcluster()
1390 pcl->partial = true; in z_erofs_decompress_pcluster()
1391 pcl->multibases = false; in z_erofs_decompress_pcluster()
1392 pcl->bvset.nextpage = NULL; in z_erofs_decompress_pcluster()
1393 pcl->vcnt = 0; in z_erofs_decompress_pcluster()
1396 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_NIL); in z_erofs_decompress_pcluster()
1397 mutex_unlock(&pcl->lock); in z_erofs_decompress_pcluster()
1418 be.pcl = container_of(owned, struct z_erofs_pcluster, next); in z_erofs_decompress_queue()
1419 owned = READ_ONCE(be.pcl->next); in z_erofs_decompress_queue()
1422 erofs_workgroup_put(&be.pcl->obj); in z_erofs_decompress_queue()
1485 static struct page *pickup_page_for_submission(struct z_erofs_pcluster *pcl, in pickup_page_for_submission() argument
1490 const pgoff_t index = pcl->obj.index; in pickup_page_for_submission()
1499 page = READ_ONCE(pcl->compressed_bvecs[nr].page); in pickup_page_for_submission()
1513 WRITE_ONCE(pcl->compressed_bvecs[nr].page, page); in pickup_page_for_submission()
1539 WRITE_ONCE(pcl->compressed_bvecs[nr].page, page); in pickup_page_for_submission()
1550 set_page_private(page, (unsigned long)pcl); in pickup_page_for_submission()
1574 if (oldpage != cmpxchg(&pcl->compressed_bvecs[nr].page, in pickup_page_for_submission()
1586 attach_page_private(page, pcl); in pickup_page_for_submission()
1631 static void move_to_bypass_jobqueue(struct z_erofs_pcluster *pcl, in move_to_bypass_jobqueue() argument
1642 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_TAIL_CLOSED); in move_to_bypass_jobqueue()
1645 WRITE_ONCE(*bypass_qtail, &pcl->next); in move_to_bypass_jobqueue()
1647 qtail[JQ_BYPASS] = &pcl->next; in move_to_bypass_jobqueue()
1708 struct z_erofs_pcluster *pcl; in z_erofs_submit_queue() local
1717 pcl = container_of(owned_head, struct z_erofs_pcluster, next); in z_erofs_submit_queue()
1720 owned_head = cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL, in z_erofs_submit_queue()
1722 if (z_erofs_is_inline_pcluster(pcl)) { in z_erofs_submit_queue()
1723 move_to_bypass_jobqueue(pcl, qtail, owned_head); in z_erofs_submit_queue()
1729 .m_pa = blknr_to_addr(pcl->obj.index), in z_erofs_submit_queue()
1734 end = cur + pcl->pclusterpages; in z_erofs_submit_queue()
1739 page = pickup_page_for_submission(pcl, i++, pagepool, in z_erofs_submit_queue()
1782 qtail[JQ_SUBMIT] = &pcl->next; in z_erofs_submit_queue()
1784 move_to_bypass_jobqueue(pcl, qtail, owned_head); in z_erofs_submit_queue()